use super::{
ToolResult, ToolResultDisplay, ToolRuntime,
args::{VIEW_IMAGE_RESPONSE_MAX_BYTES, ViewImageArgs},
contract::{metadata_key as meta, tool_name},
fs::ExistingPathPolicy,
};
use crate::{
agent::cancellation::AgentCancellation,
config::{ProviderCredential, ViewImageVisionModelSettings},
login::OPENAI_CODEX_RELOGIN_GUIDANCE,
output::redact_sensitive_text,
providers::{
ANTHROPIC_PROVIDER, CLAUDE_CODE_PROVIDER, CODEX_RESPONSES_URL, OPENAI_CODEX_PROVIDER,
ProviderEvent, StreamParser,
claude_code::{
auth::ClaudeCodeAuth,
body::{CLAUDE_CODE_BILLING_HEADER, CLAUDE_CODE_SYSTEM_PREFIX, messages_url},
},
codex_account_id_for, codex_sse_headers,
},
};
use anyhow::Context;
use base64::{Engine as _, engine::general_purpose::STANDARD};
use serde_json::{Value, json};
use std::{
collections::BTreeMap,
fs,
io::{BufReader, Read},
path::Path,
sync::{OnceLock, mpsc},
thread::{self, JoinHandle},
time::{Duration, Instant},
};
const SYSTEM_PROMPT: &str = include_str!("../../prompts/view_image_system.md");
const ANTHROPIC_MESSAGES_URL: &str = "https://api.anthropic.com/v1/messages";
const OPENAI_RESPONSES_URL: &str = "https://api.openai.com/v1/responses";
const ANTHROPIC_VERSION: &str = "2023-06-01";
const USER_AGENT: &str = concat!("magi-code/", env!("CARGO_PKG_VERSION"));
const VISION_CONNECT_TIMEOUT: Duration = Duration::from_secs(10);
const VISION_HTTP_IDLE_TIMEOUT: Duration = Duration::from_secs(90);
const VISION_ERROR_BODY_MAX_BYTES: usize = 4 * 1024;
const VISION_SUCCESS_BODY_MAX_BYTES: usize = 1024 * 1024;
const CANCEL_POLL_INTERVAL: Duration = Duration::from_millis(25);
const VISION_WORKER_JOIN_TIMEOUT: Duration = Duration::from_millis(500);
#[derive(Debug, Clone)]
struct VisionHttpRequest {
url: String,
headers: BTreeMap<String, String>,
body: String,
}
#[derive(Debug, Clone)]
enum VisionProvider {
OpenAiCompatible {
base_url: String,
api_key: Option<String>,
use_responses_endpoint: bool,
},
OpenAiCodex {
access_token: String,
account_id: String,
},
OpenAiResponses {
responses_url: String,
api_key: String,
},
ClaudeCode {
auth: ClaudeCodeAuth,
},
Anthropic {
api_key: String,
},
}
#[derive(Debug, Clone)]
struct VisionConfig {
provider: String,
model: String,
provider_kind: VisionProvider,
text_verbosity: Option<crate::config::TextVerbosity>,
}
impl ToolRuntime {
pub(super) fn view_image(
&self,
args: ViewImageArgs,
cancellation: &AgentCancellation,
) -> anyhow::Result<ToolResult> {
self.view_image_with_http(args, cancellation, &ReqwestVisionHttpClient)
}
fn view_image_with_http(
&self,
args: ViewImageArgs,
cancellation: &AgentCancellation,
http: &dyn VisionHttpClient,
) -> anyhow::Result<ToolResult> {
cancellation.check()?;
let config = self.resolve_vision_config(cancellation)?;
let max_image_bytes =
crate::config::validate_view_image_max_image_bytes(self.view_image_max_image_bytes)?;
if !self.view_image_absolute_paths {
anyhow::bail!(
"view_image is disabled because tools.view_image.absolute_paths is false"
);
}
let path = self.resolve_existing_path(
&args.path,
ExistingPathPolicy::view_image(self.view_image_absolute_paths),
)?;
let image_format = image_format_for_path(&path)?;
let media_type = image_format.media_type();
let bytes = read_bounded_image_bytes(&path, max_image_bytes)?;
let image_bytes = bytes.len();
validate_image_magic_bytes_for_path(&path, &bytes, image_format)?;
let base64 = encode_image_base64(bytes);
let request = build_vision_request(
&config,
SYSTEM_PROMPT.trim(),
&args.prompt,
media_type,
base64,
);
let text = http.post_json(request, cancellation)?;
let response = match &config.provider_kind {
VisionProvider::OpenAiCompatible {
use_responses_endpoint,
..
} => {
let value = serde_json::from_str(&text)?;
if *use_responses_endpoint {
parse_responses_response(&value)?
} else {
parse_openai_compatible_response(&value)?
}
}
VisionProvider::OpenAiCodex { .. } => parse_codex_sse_response(&text)?,
VisionProvider::OpenAiResponses { .. } => {
let value = serde_json::from_str(&text)?;
parse_responses_response(&value)?
}
VisionProvider::ClaudeCode { .. } | VisionProvider::Anthropic { .. } => {
let value = serde_json::from_str(&text)?;
parse_anthropic_response(&value)?
}
};
let sanitized_response = redact_sensitive_text(response.trim());
let (content, truncated) = truncate_utf8_response(&sanitized_response);
Ok(ToolResult {
tool_name: tool_name::VIEW_IMAGE.to_string(),
success: true,
content,
metadata: json!({
(meta::PATH): path.to_string_lossy(),
(meta::PROVIDER): config.provider,
(meta::MODEL): config.model,
(meta::MEDIA_TYPE): media_type,
(meta::IMAGE_BYTES): image_bytes,
(meta::PROMPT_CHARS): args.prompt.chars().count(),
(meta::TRUNCATED): truncated,
}),
display: ToolResultDisplay::default(),
})
}
fn resolve_vision_config(
&self,
cancellation: &AgentCancellation,
) -> anyhow::Result<VisionConfig> {
let ViewImageVisionModelSettings { provider, model } =
self.view_image_vision_model.clone().ok_or_else(|| {
anyhow::anyhow!("missing tools.view_image.vision_model configuration")
})?;
let provider = crate::config::validate_view_image_identifier("provider", &provider)?;
let model = crate::config::validate_view_image_identifier("model", &model)?;
if provider == ANTHROPIC_PROVIDER {
let paths = self
.view_image_paths
.as_ref()
.ok_or_else(|| anyhow::anyhow!("view_image auth paths are not configured"))?;
let auth = crate::config::read_auth(paths)?;
let credential = crate::config::resolve_provider_credential(
ANTHROPIC_PROVIDER,
&auth,
None,
&self.view_image_custom_providers,
)?
.ok_or_else(|| {
crate::config::ConfigError::missing_auth_for_custom_providers(
ANTHROPIC_PROVIDER,
&self.view_image_custom_providers,
&paths.auth_file,
)
})?;
let ProviderCredential::ApiKey { key } = credential else {
anyhow::bail!("view_image provider 'anthropic' requires API key auth");
};
return Ok(VisionConfig {
provider,
model,
provider_kind: VisionProvider::Anthropic { api_key: key },
text_verbosity: None,
});
}
if provider == OPENAI_CODEX_PROVIDER {
let paths = self
.view_image_paths
.as_ref()
.ok_or_else(|| anyhow::anyhow!("view_image auth paths are not configured"))?;
let credential = resolve_openai_codex_view_image_credential(paths)?;
let ProviderCredential::OAuth { access, account_id } = credential else {
anyhow::bail!("view_image provider 'openai-codex' requires OAuth auth");
};
let account_id = codex_account_id_for(&access, &account_id)?;
return Ok(VisionConfig {
provider,
model,
provider_kind: VisionProvider::OpenAiCodex {
access_token: access,
account_id,
},
text_verbosity: Some(self.view_image_codex_text_verbosity),
});
}
if provider == "openai" {
let api_key = self.resolve_openai_responses_api_key()?;
return Ok(VisionConfig {
provider,
model,
provider_kind: VisionProvider::OpenAiResponses {
responses_url: OPENAI_RESPONSES_URL.to_string(),
api_key,
},
text_verbosity: self.view_image_text_verbosity,
});
}
if provider == CLAUDE_CODE_PROVIDER {
let paths = self
.view_image_paths
.as_ref()
.ok_or_else(|| anyhow::anyhow!("view_image auth paths are not configured"))?;
let auth = crate::config::read_auth(paths)?;
let mut auth = ClaudeCodeAuth::resolve_with_api_key_fallback(&auth)?;
refresh_claude_code_view_image_auth(&mut auth, cancellation)?;
return Ok(VisionConfig {
provider,
model,
provider_kind: VisionProvider::ClaudeCode { auth },
text_verbosity: None,
});
}
let Some(custom) = self.view_image_custom_providers.get(&provider) else {
anyhow::bail!("view_image provider '{provider}' is not a configured custom provider");
};
let credential = match &custom.api_key_env_var {
Some(env_var) => std::env::var(env_var)
.ok()
.filter(|value| !value.is_empty())
.map(|key| ProviderCredential::ApiKey { key })
.ok_or_else(|| {
crate::config::ConfigError::missing_auth_for_custom_providers(
&provider,
&self.view_image_custom_providers,
self.view_image_paths
.as_ref()
.map(|paths| paths.auth_file.as_path())
.unwrap_or_else(|| std::path::Path::new("~/.magi-code/auth.json")),
)
})?,
None => ProviderCredential::NoAuth,
};
let api_key = match credential {
ProviderCredential::ApiKey { key } => Some(key),
ProviderCredential::NoAuth => None,
ProviderCredential::OAuth { .. } => {
anyhow::bail!("view_image custom provider '{provider}' requires API key or no-auth")
}
};
Ok(VisionConfig {
provider,
model,
provider_kind: VisionProvider::OpenAiCompatible {
base_url: custom.base_url.trim_end_matches('/').to_string(),
api_key,
use_responses_endpoint: custom.use_responses_endpoint,
},
text_verbosity: custom
.use_responses_endpoint
.then_some(self.view_image_text_verbosity)
.flatten()
.filter(|_| custom.supports_text_verbosity),
})
}
fn resolve_openai_responses_api_key(&self) -> anyhow::Result<String> {
if let Ok(key) = std::env::var("OPENAI_API_KEY")
&& !key.is_empty()
{
return Ok(key);
}
if let Ok(key) = std::env::var("MC_API_KEY")
&& !key.is_empty()
{
return Ok(key);
}
let paths = self
.view_image_paths
.as_ref()
.ok_or_else(|| anyhow::anyhow!("view_image auth paths are not configured"))?;
let auth = crate::config::read_auth(paths)?;
if let Some(crate::config::AuthProviderRecord::ApiKey { key }) =
auth.providers.get("openai")
&& !key.is_empty()
{
return Ok(key.clone());
}
if let Some(key) = auth.api_key.filter(|key| !key.is_empty()) {
return Ok(key);
}
Err(
crate::config::ConfigError::missing_auth_for_custom_providers(
"openai",
&self.view_image_custom_providers,
self.view_image_paths
.as_ref()
.map(|paths| paths.auth_file.as_path())
.unwrap_or_else(|| std::path::Path::new("~/.magi-code/auth.json")),
)
.into(),
)
}
}
fn resolve_openai_codex_view_image_credential(
paths: &crate::config::McPaths,
) -> anyhow::Result<ProviderCredential> {
resolve_openai_codex_view_image_credential_with_exchange(paths, crate::login::refresh_token)
}
fn resolve_openai_codex_view_image_credential_with_exchange(
paths: &crate::config::McPaths,
exchange: impl FnOnce(&str) -> anyhow::Result<crate::login::NormalizedToken>,
) -> anyhow::Result<ProviderCredential> {
let auth = crate::config::read_auth(paths)?;
let Some(record) = auth.providers.get(OPENAI_CODEX_PROVIDER).cloned() else {
anyhow::bail!("missing OAuth auth for provider 'openai-codex'; run /login openai-codex")
};
let crate::config::AuthProviderRecord::OAuth {
access,
refresh: _,
expires,
account_id,
} = record
else {
anyhow::bail!("view_image provider 'openai-codex' requires OAuth auth")
};
let near_expiry = expires.is_none_or(|expires| expires <= chrono::Utc::now().timestamp() + 300);
if !near_expiry && !access.is_empty() {
return Ok(ProviderCredential::OAuth { access, account_id });
}
crate::login::codex_credential_from_store_with_exchange(paths, exchange).map_err(|error| {
anyhow::anyhow!(
"view_image provider 'openai-codex' OAuth refresh failed; {OPENAI_CODEX_RELOGIN_GUIDANCE}: {error}"
)
})
}
fn refresh_claude_code_view_image_auth(
auth: &mut ClaudeCodeAuth,
cancellation: &AgentCancellation,
) -> anyhow::Result<()> {
auth.refresh_if_needed(cancellation).map_err(|error| {
anyhow::anyhow!("view_image provider 'claude-code' OAuth refresh failed: {error}")
})
}
fn build_vision_request(
config: &VisionConfig,
system_prompt: &str,
prompt: &str,
media_type: &str,
base64: String,
) -> VisionHttpRequest {
match &config.provider_kind {
VisionProvider::OpenAiCompatible {
base_url,
api_key,
use_responses_endpoint,
} => {
let mut headers = json_headers();
if let Some(api_key) = api_key {
headers.insert("authorization".to_string(), format!("Bearer {api_key}"));
}
if *use_responses_endpoint {
return VisionHttpRequest {
url: format!("{base_url}/responses"),
headers,
body: responses_view_image_body(
&config.model,
system_prompt,
prompt,
media_type,
base64,
config.text_verbosity,
),
};
}
VisionHttpRequest {
url: format!("{base_url}/chat/completions"),
headers,
body: chat_completions_view_image_body(
&config.model,
system_prompt,
prompt,
media_type,
base64,
),
}
}
VisionProvider::OpenAiCodex {
access_token,
account_id,
} => VisionHttpRequest {
url: CODEX_RESPONSES_URL.to_string(),
headers: codex_sse_headers(access_token, account_id),
body: codex_view_image_body(
&config.model,
system_prompt,
prompt,
media_type,
base64,
config.text_verbosity,
),
},
VisionProvider::OpenAiResponses {
responses_url,
api_key,
} => {
let mut headers = json_headers();
headers.insert("authorization".to_string(), format!("Bearer {api_key}"));
VisionHttpRequest {
url: responses_url.clone(),
headers,
body: responses_view_image_body(
&config.model,
system_prompt,
prompt,
media_type,
base64,
config.text_verbosity,
),
}
}
VisionProvider::ClaudeCode { auth } => {
let mut headers =
crate::providers::claude_code::headers::claude_code_headers(auth.access_mode());
headers.insert("accept".to_string(), "application/json".to_string());
let system = if auth.access_mode().is_oauth() {
format!(
"{CLAUDE_CODE_BILLING_HEADER}\n\n{CLAUDE_CODE_SYSTEM_PREFIX}\n\n{system_prompt}"
)
} else {
format!("{CLAUDE_CODE_SYSTEM_PREFIX}\n\n{system_prompt}")
};
VisionHttpRequest {
url: messages_url(),
headers,
body: anthropic_view_image_body(
&crate::model_catalog::resolve_claude_code_model_alias(&config.model),
Some(&system),
prompt,
media_type,
base64,
),
}
}
VisionProvider::Anthropic { api_key } => {
let mut headers = json_headers();
headers.insert("x-api-key".to_string(), api_key.clone());
headers.insert(
"anthropic-version".to_string(),
ANTHROPIC_VERSION.to_string(),
);
VisionHttpRequest {
url: ANTHROPIC_MESSAGES_URL.to_string(),
headers,
body: anthropic_view_image_body(
&config.model,
Some(system_prompt),
prompt,
media_type,
base64,
),
}
}
}
}
fn encode_image_base64(bytes: Vec<u8>) -> String {
let mut base64 = String::with_capacity(bytes.len().div_ceil(3) * 4);
STANDARD.encode_string(bytes, &mut base64);
base64
}
fn chat_completions_view_image_body(
model: &str,
system_prompt: &str,
prompt: &str,
media_type: &str,
base64: String,
) -> String {
let model = json_string(model);
let system_prompt = json_string(system_prompt);
let prompt = json_string(prompt);
let mut body = String::with_capacity(
180 + model.len() + system_prompt.len() + prompt.len() + media_type.len() + base64.len(),
);
body.push_str("{\"model\":");
body.push_str(&model);
body.push_str(",\"stream\":false,\"messages\":[{\"role\":\"system\",\"content\":");
body.push_str(&system_prompt);
body.push_str("},{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":");
body.push_str(&prompt);
body.push_str("},{\"type\":\"image_url\",\"image_url\":{\"url\":");
push_data_url_json_string(&mut body, media_type, &base64);
body.push_str("}}]}]}");
body
}
fn codex_view_image_body(
model: &str,
system_prompt: &str,
prompt: &str,
media_type: &str,
base64: String,
text_verbosity: Option<crate::config::TextVerbosity>,
) -> String {
let model = json_string(model);
let system_prompt = json_string(system_prompt);
let prompt = json_string(prompt);
let mut body = String::with_capacity(
240 + model.len() + system_prompt.len() + prompt.len() + media_type.len() + base64.len(),
);
body.push_str("{\"model\":");
body.push_str(&model);
body.push_str(",\"store\":false,\"stream\":true,\"instructions\":");
body.push_str(&system_prompt);
body.push_str(",\"input\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_text\",\"text\":");
body.push_str(&prompt);
body.push_str("},{\"type\":\"input_image\",\"image_url\":");
push_data_url_json_string(&mut body, media_type, &base64);
body.push_str("}]}]");
if let Some(text_verbosity) = text_verbosity {
body.push_str(",\"text\":{\"verbosity\":");
body.push_str(&json_string(text_verbosity.as_api_str()));
body.push('}');
}
body.push_str(",\"include\":[\"reasoning.encrypted_content\"]}");
body
}
fn responses_view_image_body(
model: &str,
system_prompt: &str,
prompt: &str,
media_type: &str,
base64: String,
text_verbosity: Option<crate::config::TextVerbosity>,
) -> String {
let model = json_string(model);
let system_prompt = json_string(system_prompt);
let prompt = json_string(prompt);
let mut body = String::with_capacity(
180 + model.len() + system_prompt.len() + prompt.len() + media_type.len() + base64.len(),
);
body.push_str("{\"model\":");
body.push_str(&model);
body.push_str(",\"stream\":false,\"instructions\":");
body.push_str(&system_prompt);
body.push_str(",\"input\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_text\",\"text\":");
body.push_str(&prompt);
body.push_str("},{\"type\":\"input_image\",\"image_url\":");
push_data_url_json_string(&mut body, media_type, &base64);
body.push_str("}]}]");
if let Some(text_verbosity) = text_verbosity {
body.push_str(",\"text\":{\"verbosity\":");
body.push_str(&json_string(text_verbosity.as_api_str()));
body.push('}');
}
body.push('}');
body
}
fn anthropic_view_image_body(
model: &str,
system_prompt: Option<&str>,
prompt: &str,
media_type: &str,
base64: String,
) -> String {
let model = json_string(model);
let prompt = json_string(prompt);
let system_prompt = system_prompt.map(json_string);
let mut body = String::with_capacity(
220 + model.len()
+ system_prompt.as_ref().map_or(0, String::len)
+ prompt.len()
+ media_type.len()
+ base64.len(),
);
body.push_str("{\"model\":");
body.push_str(&model);
body.push_str(",\"max_tokens\":4096,\"stream\":false");
if let Some(system_prompt) = system_prompt {
body.push_str(",\"system\":");
body.push_str(&system_prompt);
}
body.push_str(",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"image\",\"source\":{\"type\":\"base64\",\"media_type\":");
push_json_string_unescaped(&mut body, media_type);
body.push_str(",\"data\":");
push_json_string_unescaped(&mut body, &base64);
body.push_str("}},{\"type\":\"text\",\"text\":");
body.push_str(&prompt);
body.push_str("}]}]}");
body
}
fn json_string(value: &str) -> String {
serde_json::to_string(value).expect("serializing string to JSON cannot fail")
}
fn push_data_url_json_string(body: &mut String, media_type: &str, base64: &str) {
body.push('"');
body.push_str("data:");
body.push_str(media_type);
body.push_str(";base64,");
body.push_str(base64);
body.push('"');
}
fn push_json_string_unescaped(body: &mut String, value: &str) {
body.push('"');
body.push_str(value);
body.push('"');
}
fn json_headers() -> BTreeMap<String, String> {
BTreeMap::from([
("accept".to_string(), "application/json".to_string()),
("content-type".to_string(), "application/json".to_string()),
("user-agent".to_string(), USER_AGENT.to_string()),
])
}
trait VisionHttpClient {
fn post_json(
&self,
request: VisionHttpRequest,
cancellation: &AgentCancellation,
) -> anyhow::Result<String>;
}
#[derive(Debug, Clone, Copy)]
struct ReqwestVisionHttpClient;
impl VisionHttpClient for ReqwestVisionHttpClient {
fn post_json(
&self,
request: VisionHttpRequest,
cancellation: &AgentCancellation,
) -> anyhow::Result<String> {
post_json(request, cancellation)
}
}
fn post_json(
request: VisionHttpRequest,
cancellation: &AgentCancellation,
) -> anyhow::Result<String> {
cancellation.check()?;
let client = vision_http_client()?;
let mut builder = client.post(&request.url);
for (name, value) in request.headers {
builder = builder.header(name, value);
}
let url = request.url.clone();
let response = run_cancellable(
"vision provider response header timeout",
VISION_HTTP_IDLE_TIMEOUT,
cancellation,
move || {
builder
.body(request.body)
.send()
.map_err(anyhow::Error::from)
},
)?;
let status = response.status();
let body = read_bounded_body_cancellable(
response,
if status.is_success() {
VISION_SUCCESS_BODY_MAX_BYTES
} else {
VISION_ERROR_BODY_MAX_BYTES
},
VISION_HTTP_IDLE_TIMEOUT,
cancellation,
)?;
let text = String::from_utf8_lossy(&body).into_owned();
if !status.is_success() {
anyhow::bail!(
"vision provider request failed for {} with status {status}: {}",
sanitize_url(&url),
redact_sensitive_text(&text)
);
}
Ok(text)
}
fn vision_http_client() -> anyhow::Result<&'static reqwest::blocking::Client> {
static CLIENT: OnceLock<anyhow::Result<reqwest::blocking::Client, String>> = OnceLock::new();
CLIENT
.get_or_init(|| {
reqwest::blocking::Client::builder()
.connect_timeout(VISION_CONNECT_TIMEOUT)
.timeout(VISION_HTTP_IDLE_TIMEOUT)
.build()
.map_err(|error| error.to_string())
})
.as_ref()
.map_err(|error| anyhow::anyhow!(error.clone()))
}
struct VisionWorkerHandle {
name: &'static str,
done_receiver: mpsc::Receiver<()>,
join_handle: JoinHandle<()>,
}
impl VisionWorkerHandle {
fn join_or_warn(self) {
match self.done_receiver.recv_timeout(VISION_WORKER_JOIN_TIMEOUT) {
Ok(()) | Err(mpsc::RecvTimeoutError::Disconnected) => {
if self.join_handle.join().is_err() {
eprintln!("magi-code warning: {} worker panicked", self.name);
}
}
Err(mpsc::RecvTimeoutError::Timeout) => {
eprintln!(
"magi-code warning: {} worker did not exit within cleanup grace period; detaching worker",
self.name
);
}
}
}
}
fn spawn_vision_worker<T, F>(
name: &'static str,
operation: F,
) -> anyhow::Result<(mpsc::Receiver<anyhow::Result<T>>, VisionWorkerHandle)>
where
T: Send + 'static,
F: FnOnce() -> anyhow::Result<T> + Send + 'static,
{
let (sender, receiver) = mpsc::sync_channel(1);
let (done_sender, done_receiver) = mpsc::sync_channel(1);
let join_handle = thread::Builder::new()
.name(name.to_string())
.spawn(move || {
let _ = sender.send(operation());
let _ = done_sender.send(());
})?;
Ok((
receiver,
VisionWorkerHandle {
name,
done_receiver,
join_handle,
},
))
}
fn run_cancellable<T, F>(
timeout_label: &'static str,
idle_timeout: Duration,
cancellation: &AgentCancellation,
operation: F,
) -> anyhow::Result<T>
where
T: Send + 'static,
F: FnOnce() -> anyhow::Result<T> + Send + 'static,
{
let (receiver, worker) = spawn_vision_worker("view-image-http", operation)?;
let result = recv_cancellable(&receiver, timeout_label, idle_timeout, cancellation);
if result.is_err() {
drop(receiver);
}
worker.join_or_warn();
result?
}
fn recv_cancellable<T>(
receiver: &mpsc::Receiver<T>,
timeout_label: &str,
idle_timeout: Duration,
cancellation: &AgentCancellation,
) -> anyhow::Result<T> {
let start = Instant::now();
loop {
cancellation.check()?;
if start.elapsed() >= idle_timeout {
anyhow::bail!("{timeout_label} after {}s", idle_timeout.as_secs());
}
let remaining = idle_timeout.saturating_sub(start.elapsed());
match receiver.recv_timeout(remaining.min(CANCEL_POLL_INTERVAL)) {
Ok(value) => return Ok(value),
Err(mpsc::RecvTimeoutError::Timeout) => continue,
Err(mpsc::RecvTimeoutError::Disconnected) => {
cancellation.check()?;
anyhow::bail!("vision provider HTTP worker disconnected");
}
}
}
}
fn read_bounded_body_cancellable<R>(
mut reader: R,
max_bytes: usize,
idle_timeout: Duration,
cancellation: &AgentCancellation,
) -> anyhow::Result<Vec<u8>>
where
R: Read + Send + 'static,
{
let (sender, receiver) = mpsc::sync_channel::<std::io::Result<Vec<u8>>>(1);
let (done_sender, done_receiver) = mpsc::sync_channel(1);
let join_handle = thread::Builder::new()
.name("view-image-body".to_string())
.spawn(move || {
let mut buffer = [0_u8; 8192];
loop {
let message = match reader.read(&mut buffer) {
Ok(0) => Ok(Vec::new()),
Ok(read) => Ok(buffer[..read].to_vec()),
Err(error) => Err(error),
};
let done = matches!(&message, Ok(bytes) if bytes.is_empty()) || message.is_err();
if sender.send(message).is_err() || done {
break;
}
}
let _ = done_sender.send(());
})?;
let worker = VisionWorkerHandle {
name: "view-image-body",
done_receiver,
join_handle,
};
let mut body = Vec::with_capacity(max_bytes);
loop {
let message = match recv_cancellable(
&receiver,
"vision provider response body timeout",
idle_timeout,
cancellation,
) {
Ok(message) => message,
Err(error) => {
drop(receiver);
worker.join_or_warn();
return Err(error);
}
};
match message {
Ok(bytes) if bytes.is_empty() => {
worker.join_or_warn();
return Ok(body);
}
Ok(bytes) => {
let remaining = max_bytes.saturating_sub(body.len());
if bytes.len() > remaining {
body.extend_from_slice(&bytes[..remaining]);
drop(receiver);
worker.join_or_warn();
return Ok(body);
}
body.extend_from_slice(&bytes);
}
Err(error) => {
worker.join_or_warn();
return Err(error.into());
}
}
}
}
fn sanitize_url(url: &str) -> String {
let Ok(mut parsed) = reqwest::Url::parse(url) else {
return redact_sensitive_text(url);
};
let _ = parsed.set_username("");
let _ = parsed.set_password(None);
parsed.set_query(None);
parsed.set_fragment(None);
parsed.to_string()
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ImageFormat {
Png,
Jpeg,
Gif,
Webp,
}
impl ImageFormat {
fn media_type(&self) -> &'static str {
match self {
ImageFormat::Png => "image/png",
ImageFormat::Jpeg => "image/jpeg",
ImageFormat::Gif => "image/gif",
ImageFormat::Webp => "image/webp",
}
}
}
fn image_format_for_path(path: &Path) -> anyhow::Result<ImageFormat> {
let extension = path
.extension()
.and_then(|value| value.to_str())
.unwrap_or_default()
.to_ascii_lowercase();
match extension.as_str() {
"png" => Ok(ImageFormat::Png),
"jpg" | "jpeg" => Ok(ImageFormat::Jpeg),
"gif" => Ok(ImageFormat::Gif),
"webp" => Ok(ImageFormat::Webp),
_ => anyhow::bail!("unsupported image type; expected png, jpg, jpeg, gif, or webp"),
}
}
fn read_bounded_image_bytes(path: &Path, max_bytes: u64) -> anyhow::Result<Vec<u8>> {
let file = fs::File::open(path)
.with_context(|| format!("failed to open image file '{}'", path.display()))?;
let metadata = file
.metadata()
.with_context(|| format!("failed to read image metadata for '{}'", path.display()))?;
if !metadata.is_file() {
anyhow::bail!("image path must be a regular file: {}", path.display());
}
let read_limit = max_bytes.saturating_add(1);
if metadata.len() > read_limit {
anyhow::bail!(
"image file is {} bytes, exceeding configured maximum of {} bytes",
metadata.len(),
max_bytes
);
}
let mut reader = BufReader::new(file).take(read_limit);
let mut bytes = Vec::new();
reader
.read_to_end(&mut bytes)
.with_context(|| format!("failed to read image file '{}'", path.display()))?;
if u64::try_from(bytes.len()).unwrap_or(u64::MAX) > max_bytes {
anyhow::bail!(
"image file is {} bytes, exceeding configured maximum of {} bytes",
bytes.len(),
max_bytes
);
}
Ok(bytes)
}
fn validate_image_magic_bytes_for_path(
path: &Path,
bytes: &[u8],
expected: ImageFormat,
) -> anyhow::Result<()> {
let matches = match expected {
ImageFormat::Png => bytes.starts_with(&[0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A]),
ImageFormat::Jpeg => bytes.starts_with(&[0xFF, 0xD8, 0xFF]),
ImageFormat::Gif => bytes.starts_with(b"GIF87a") || bytes.starts_with(b"GIF89a"),
ImageFormat::Webp => bytes.starts_with(b"RIFF") && bytes.get(8..12) == Some(b"WEBP"),
};
if !matches {
anyhow::bail!(
"file extension '{}' does not match file content (expected {} image data)",
path.extension().and_then(|e| e.to_str()).unwrap_or("?"),
expected.media_type()
);
}
Ok(())
}
#[cfg(test)]
pub(crate) fn media_type_for_path(path: &Path) -> anyhow::Result<&'static str> {
let extension = path
.extension()
.and_then(|value| value.to_str())
.unwrap_or_default()
.to_ascii_lowercase();
match extension.as_str() {
"png" => Ok("image/png"),
"jpg" | "jpeg" => Ok("image/jpeg"),
"gif" => Ok("image/gif"),
"webp" => Ok("image/webp"),
_ => anyhow::bail!("unsupported image type; expected png, jpg, jpeg, gif, or webp"),
}
}
fn parse_openai_compatible_response(value: &Value) -> anyhow::Result<String> {
let text = value
.pointer("/choices/0/message/content")
.and_then(Value::as_str)
.map(str::trim)
.filter(|text| !text.is_empty())
.ok_or_else(|| {
anyhow::anyhow!("vision provider response missing choices[0].message.content")
})?;
Ok(text.to_string())
}
fn parse_responses_response(value: &Value) -> anyhow::Result<String> {
let output = value
.get("output")
.and_then(Value::as_array)
.ok_or_else(|| anyhow::anyhow!("responses vision response missing output array"))?;
let text = output
.iter()
.filter_map(|item| item.get("content").and_then(Value::as_array))
.flatten()
.filter(|block| block.get("type").and_then(Value::as_str) == Some("output_text"))
.filter_map(|block| block.get("text").and_then(Value::as_str))
.collect::<Vec<_>>()
.join("");
let text = text.trim();
if text.is_empty() {
anyhow::bail!("responses vision response contained no output_text content");
}
Ok(text.to_string())
}
fn parse_codex_sse_response(text: &str) -> anyhow::Result<String> {
let mut parser = StreamParser::default();
let mut output = String::new();
for event in parser.push_chunk_outcome(text)?.events {
if let ProviderEvent::TextDelta(delta) = event {
output.push_str(&delta);
}
}
for event in parser.finish()? {
if let ProviderEvent::TextDelta(delta) = event {
output.push_str(&delta);
}
}
let output = output.trim();
if output.is_empty() {
anyhow::bail!("codex vision SSE response contained no output text");
}
Ok(output.to_string())
}
fn parse_anthropic_response(value: &Value) -> anyhow::Result<String> {
let content = value
.get("content")
.and_then(Value::as_array)
.ok_or_else(|| anyhow::anyhow!("anthropic vision response missing content array"))?;
let text = content
.iter()
.filter(|block| block.get("type").and_then(Value::as_str) == Some("text"))
.filter_map(|block| block.get("text").and_then(Value::as_str))
.collect::<Vec<_>>()
.join("");
let text = text.trim();
if text.is_empty() {
anyhow::bail!("anthropic vision response contained no text");
}
Ok(text.to_string())
}
fn truncate_utf8_response(text: &str) -> (String, bool) {
if text.len() <= VIEW_IMAGE_RESPONSE_MAX_BYTES {
return (text.to_string(), false);
}
const MARKER: &str = "\n\n[view_image response truncated at 65536 bytes]";
let cap = VIEW_IMAGE_RESPONSE_MAX_BYTES.saturating_sub(MARKER.len());
let mut end = cap;
while !text.is_char_boundary(end) {
end -= 1;
}
let mut truncated = text[..end].to_string();
truncated.push_str(MARKER);
(truncated, true)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{
agent::cancellation::{AgentCancellation, AgentCancellationHandle},
config::{CustomProviderConfig, ToolSettings, ViewImageToolSettings},
tools::ToolRuntime,
};
use base64::Engine;
use std::{
cell::RefCell,
collections::BTreeMap,
fs,
io::{Read, Write},
net::TcpListener,
path::PathBuf,
sync::{Arc, Mutex, mpsc},
time::{Duration, Instant},
};
const PNG_BYTES: &[u8] = &[0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A];
#[derive(Default)]
struct FakeVisionHttpClient {
requests: RefCell<Vec<VisionHttpRequest>>,
response: RefCell<String>,
}
impl FakeVisionHttpClient {
fn with_response(response: Value) -> Self {
Self::with_text(response.to_string())
}
fn with_text(response: impl Into<String>) -> Self {
Self {
requests: RefCell::new(Vec::new()),
response: RefCell::new(response.into()),
}
}
fn request_count(&self) -> usize {
self.requests.borrow().len()
}
}
impl VisionHttpClient for FakeVisionHttpClient {
fn post_json(
&self,
request: VisionHttpRequest,
_cancellation: &AgentCancellation,
) -> anyhow::Result<String> {
self.requests.borrow_mut().push(request);
Ok(self.response.borrow().clone())
}
}
fn request_body_json(request: &VisionHttpRequest) -> Value {
serde_json::from_str(&request.body).unwrap()
}
fn runtime(temp: &tempfile::TempDir, provider: &str, max_image_bytes: u64) -> ToolRuntime {
let mut runtime = ToolRuntime::new_with_settings(
temp.path(),
ToolSettings {
view_image: ViewImageToolSettings {
vision_model: Some(ViewImageVisionModelSettings {
provider: provider.to_string(),
model: "vision-model".to_string(),
}),
absolute_paths: true,
max_image_bytes,
},
..ToolSettings::default()
},
)
.unwrap();
runtime.view_image_custom_providers = BTreeMap::from([
(
"local-vision".to_string(),
CustomProviderConfig {
label: "Local Vision".to_string(),
base_url: "http://localhost:11434/v1".to_string(),
api_key_env_var: None,
models_dev_provider: None,
use_responses_endpoint: false,
supports_text_verbosity: false,
reasoning_protocol: crate::config::CustomReasoningProtocol::default(),
extra_models: Vec::new(),
},
),
(
"local-responses".to_string(),
CustomProviderConfig {
label: "Local Responses".to_string(),
base_url: "http://localhost:11434/v1".to_string(),
api_key_env_var: None,
models_dev_provider: None,
use_responses_endpoint: true,
supports_text_verbosity: false,
reasoning_protocol: crate::config::CustomReasoningProtocol::default(),
extra_models: Vec::new(),
},
),
]);
runtime
}
fn write_image(temp: &tempfile::TempDir, name: &str, bytes: &[u8]) -> PathBuf {
let path = temp.path().join(name);
fs::write(&path, bytes).unwrap();
path
}
fn view_args(path: &Path) -> ViewImageArgs {
ViewImageArgs {
path: path.display().to_string(),
prompt: "describe image".to_string(),
}
}
fn openai_response(text: &str) -> Value {
json!({"choices":[{"message":{"content":text}}]})
}
fn responses_response(text: &str) -> Value {
json!({"output":[{"type":"message","content":[{"type":"output_text","text":text}]}]})
}
fn codex_sse_response(text: &str) -> String {
format!(
"data: {{\"type\":\"response.output_text.delta\",\"delta\":{}}}\n\ndata: {{\"type\":\"response.completed\",\"response\":{{}}}}\n\n",
serde_json::to_string(text).unwrap()
)
}
fn fake_jwt(account_id: &str) -> String {
let header = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(r#"{"alg":"none"}"#);
let payload = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(format!(
r#"{{"https://api.openai.com/auth.chatgpt_account_id":"{account_id}"}}"#
));
format!("{header}.{payload}.")
}
#[test]
fn media_type_maps_supported_extensions_case_insensitive() {
assert_eq!(
media_type_for_path(Path::new("a.PNG")).unwrap(),
"image/png"
);
assert_eq!(
media_type_for_path(Path::new("a.jpg")).unwrap(),
"image/jpeg"
);
assert_eq!(
media_type_for_path(Path::new("a.jpeg")).unwrap(),
"image/jpeg"
);
assert_eq!(
media_type_for_path(Path::new("a.gif")).unwrap(),
"image/gif"
);
assert_eq!(
media_type_for_path(Path::new("a.webp")).unwrap(),
"image/webp"
);
assert!(media_type_for_path(Path::new("a.txt")).is_err());
}
#[test]
fn read_bounded_image_bytes_rejects_oversized_file() {
let temp = tempfile::TempDir::new().unwrap();
let path = temp.path().join("big.png");
let mut content = PNG_BYTES.to_vec();
content.extend(std::iter::repeat_n(0u8, 100));
std::fs::write(&path, &content).unwrap();
let result = read_bounded_image_bytes(&path, 10);
assert!(result.is_err());
assert!(result.unwrap_err().to_string().contains("exceeding"));
}
#[test]
fn magic_bytes_reject_non_image_with_image_extension() {
let temp = tempfile::TempDir::new().unwrap();
let path = temp.path().join("fake.png");
std::fs::write(&path, b"this is not a PNG file").unwrap();
let format = image_format_for_path(&path).unwrap();
let bytes = read_bounded_image_bytes(&path, 1024 * 1024).unwrap();
let result = validate_image_magic_bytes_for_path(&path, &bytes, format);
assert!(result.is_err());
assert!(result.unwrap_err().to_string().contains("does not match"));
}
#[test]
fn magic_bytes_accept_valid_png_signature() {
let temp = tempfile::TempDir::new().unwrap();
let path = temp.path().join("valid.png");
std::fs::write(&path, PNG_BYTES).unwrap();
let format = image_format_for_path(&path).unwrap();
let bytes = read_bounded_image_bytes(&path, 1024 * 1024).unwrap();
assert!(validate_image_magic_bytes_for_path(&path, &bytes, format).is_ok());
}
#[test]
fn truncated_output_respects_max_bytes_including_marker() {
let long = "x".repeat(VIEW_IMAGE_RESPONSE_MAX_BYTES + 1000);
let (result, truncated) = truncate_utf8_response(&long);
assert!(truncated);
assert!(result.len() <= VIEW_IMAGE_RESPONSE_MAX_BYTES);
}
#[test]
fn read_bounded_body_cancellable_returns_prompt_canceled_without_waiting_for_blocked_reader() {
struct BlockingRead(mpsc::Receiver<()>);
impl Read for BlockingRead {
fn read(&mut self, _buffer: &mut [u8]) -> std::io::Result<usize> {
let _ = self.0.recv();
Ok(0)
}
}
let (block_sender, block_receiver) = mpsc::channel();
let (cancellation, handle): (AgentCancellation, AgentCancellationHandle) =
AgentCancellation::default().child_token();
handle.cancel();
let start = Instant::now();
let result = read_bounded_body_cancellable(
BlockingRead(block_receiver),
1024,
Duration::from_secs(30),
&cancellation,
);
drop(block_sender);
assert!(result.is_err());
assert_eq!(result.unwrap_err().to_string(), "prompt canceled");
assert!(start.elapsed() < Duration::from_secs(2));
}
#[test]
fn parsers_reject_empty_and_extract_text() {
assert_eq!(
parse_openai_compatible_response(&json!({"choices":[{"message":{"content":" ok "}}]}))
.unwrap(),
"ok"
);
assert!(parse_openai_compatible_response(&json!({"choices":[]})).is_err());
assert_eq!(
parse_responses_response(&json!({"output":[
{"type":"message","content":[{"type":"output_text","text":" hello"}]},
{"type":"message","content":[{"type":"output_text","text":" world "}]}
]}))
.unwrap(),
"hello world"
);
assert!(parse_responses_response(&json!({"output":[]})).is_err());
assert_eq!(
parse_codex_sse_response(concat!(
"data: {\"type\":\"response.output_text.delta\",\"delta\":\"hello\"}\n\n",
"data: {\"type\":\"response.output_text.delta\",\"delta\":\" world\"}\n\n",
"data: {\"type\":\"response.completed\",\"response\":{}}\n\n"
))
.unwrap(),
"hello world"
);
assert!(parse_codex_sse_response("data: {bad}\n\n").is_err());
assert!(
parse_codex_sse_response(
"data: {\"type\":\"response.output_text.delta\",\"delta\":\"partial\"}\n\n"
)
.unwrap_err()
.to_string()
.contains("missing provider stream completion")
);
assert!(
parse_codex_sse_response("data: {\"type\":\"response.completed\"}\n\n")
.unwrap_err()
.to_string()
.contains("no output text")
);
assert_eq!(
parse_anthropic_response(&json!({"content":[{"type":"text","text":"hello"},{"type":"image","x":1},{"type":"text","text":" world"}]})).unwrap(),
"hello world"
);
assert!(
parse_anthropic_response(&json!({"content":[{"type":"text","text":" "}]})).is_err()
);
}
#[test]
fn truncates_at_utf8_boundary() {
let text = format!("{}☃", "a".repeat(VIEW_IMAGE_RESPONSE_MAX_BYTES));
let (truncated, was_truncated) = truncate_utf8_response(&text);
assert!(was_truncated);
assert!(truncated.contains("[view_image response truncated at 65536 bytes]"));
assert!(truncated.is_char_boundary(VIEW_IMAGE_RESPONSE_MAX_BYTES));
}
#[test]
fn validation_and_config_errors_happen_before_http() {
let temp = tempfile::TempDir::new().unwrap();
let fake = FakeVisionHttpClient::with_response(openai_response("ok"));
let image = write_image(&temp, "image.png", b"image-bytes");
let missing_config = ToolRuntime::new(temp.path()).unwrap();
assert!(
missing_config
.view_image_with_http(view_args(&image), &AgentCancellation::default(), &fake)
.unwrap_err()
.to_string()
.contains("vision_model")
);
let rt = runtime(&temp, "local-vision", 8);
assert!(
rt.view_image_with_http(
ViewImageArgs {
path: "relative.png".to_string(),
prompt: "p".to_string(),
},
&AgentCancellation::default(),
&fake,
)
.unwrap_err()
.to_string()
.contains("No such file")
);
let mut disabled = runtime(&temp, "local-vision", 8);
disabled.view_image_absolute_paths = false;
assert!(
disabled
.view_image_with_http(view_args(&image), &AgentCancellation::default(), &fake)
.unwrap_err()
.to_string()
.contains("absolute_paths is false")
);
let image_dir = temp.path().join("dir.png");
fs::create_dir(&image_dir).unwrap();
assert!(
rt.view_image_with_http(view_args(&image_dir), &AgentCancellation::default(), &fake)
.unwrap_err()
.to_string()
.contains("regular file")
);
assert!(
rt.view_image_with_http(
view_args(&temp.path().join("missing.png")),
&AgentCancellation::default(),
&fake,
)
.unwrap_err()
.to_string()
.contains("No such file")
);
let txt = write_image(&temp, "bad.txt", b"x");
assert!(
rt.view_image_with_http(view_args(&txt), &AgentCancellation::default(), &fake)
.unwrap_err()
.to_string()
.contains("unsupported image type")
);
assert!(
rt.view_image_with_http(view_args(&image), &AgentCancellation::default(), &fake)
.unwrap_err()
.to_string()
.contains("exceeding")
);
assert_eq!(fake.request_count(), 0);
}
#[test]
fn fake_http_openai_request_contains_required_body_and_result_excludes_request() {
let temp = tempfile::TempDir::new().unwrap();
let image = write_image(&temp, "image.png", PNG_BYTES);
let runtime = runtime(&temp, "local-vision", 1024);
let fake = FakeVisionHttpClient::with_response(openai_response("visible answer"));
let result = runtime
.view_image_with_http(view_args(&image), &AgentCancellation::default(), &fake)
.unwrap();
let requests = fake.requests.borrow();
let body = request_body_json(&requests[0]);
let body_text = &requests[0].body;
assert_eq!(body["messages"][0]["content"], SYSTEM_PROMPT.trim());
assert!(body_text.contains("describe image"));
assert!(body_text.contains("data:image/png;base64,iVBORw0KGgo="));
assert_eq!(body["stream"], false);
assert_eq!(body["model"], "vision-model");
let surfaces = format!(
"{}{}{}",
result.content,
result.metadata,
serde_json::to_string(&result).unwrap()
);
assert!(!surfaces.contains("data:image"), "{surfaces}");
assert!(!surfaces.contains("iVBORw0KGgo="), "{surfaces}");
assert!(!surfaces.contains("messages"), "{surfaces}");
}
#[test]
fn custom_provider_with_use_responses_endpoint_hits_responses_url() {
let temp = tempfile::TempDir::new().unwrap();
let image = write_image(&temp, "image.png", PNG_BYTES);
let runtime = runtime(&temp, "local-responses", 1024);
let fake = FakeVisionHttpClient::with_response(responses_response("responses ok"));
let result = runtime
.view_image_with_http(view_args(&image), &AgentCancellation::default(), &fake)
.unwrap();
let requests = fake.requests.borrow();
assert_eq!(result.content, "responses ok");
assert_eq!(requests[0].url, "http://localhost:11434/v1/responses");
let body = request_body_json(&requests[0]);
assert_eq!(body["input"][0]["content"][1]["type"], "input_image");
}
#[test]
fn anthropic_request_body_contains_required_fields() {
let config = VisionConfig {
provider: "anthropic".to_string(),
model: "claude-vision".to_string(),
provider_kind: VisionProvider::Anthropic {
api_key: "secret-key".to_string(),
},
text_verbosity: None,
};
let request = build_vision_request(
&config,
"system prompt",
"user prompt",
"image/jpeg",
"QUJD".to_string(),
);
let body = request_body_json(&request);
let body_text = &request.body;
assert!(body_text.contains("system prompt"));
assert!(body_text.contains("user prompt"));
assert!(body_text.contains("image/jpeg"));
assert!(body_text.contains("QUJD"));
assert_eq!(body["stream"], false);
assert_eq!(body["max_tokens"], 4096);
assert_eq!(body["model"], "claude-vision");
}
#[test]
fn responses_request_body_contains_required_fields() {
let config = VisionConfig {
provider: "openai".to_string(),
model: "gpt-vision".to_string(),
provider_kind: VisionProvider::OpenAiResponses {
responses_url: OPENAI_RESPONSES_URL.to_string(),
api_key: "secret-key".to_string(),
},
text_verbosity: None,
};
let request = build_vision_request(
&config,
"system prompt",
"user prompt",
"image/png",
"QUJD".to_string(),
);
let body = request_body_json(&request);
assert_eq!(request.url, OPENAI_RESPONSES_URL);
assert_eq!(request.headers["authorization"], "Bearer secret-key");
assert_eq!(body["instructions"], "system prompt");
assert_eq!(body["stream"], false);
assert_eq!(body["model"], "gpt-vision");
assert_eq!(body["input"][0]["role"], "user");
assert_eq!(body["input"][0]["content"][0]["type"], "input_text");
assert_eq!(body["input"][0]["content"][0]["text"], "user prompt");
assert_eq!(body["input"][0]["content"][1]["type"], "input_image");
assert_eq!(
body["input"][0]["content"][1]["image_url"],
"data:image/png;base64,QUJD"
);
}
#[test]
fn responses_request_body_serializes_each_optional_text_verbosity() {
for (verbosity, expected) in [
(crate::config::TextVerbosity::Low, "low"),
(crate::config::TextVerbosity::Medium, "medium"),
(crate::config::TextVerbosity::High, "high"),
] {
let body = responses_view_image_body(
"gpt-vision",
"system",
"prompt",
"image/png",
"QUJD".to_string(),
Some(verbosity),
);
let body = serde_json::from_str::<Value>(&body).unwrap();
assert_eq!(body["text"]["verbosity"], expected);
}
let body = responses_view_image_body(
"gpt-vision",
"system",
"prompt",
"image/png",
"QUJD".to_string(),
None,
);
let body = serde_json::from_str::<Value>(&body).unwrap();
assert!(body.get("text").is_none());
}
#[test]
fn unsupported_providers_reject_without_consuming_global_api_keys_or_selected_model() {
let env = crate::test_support::env::env_lock();
env.set_var("MC_API_KEY", "mc-secret");
env.set_var("OPENAI_API_KEY", "openai-secret");
let temp = tempfile::TempDir::new().unwrap();
let fake = FakeVisionHttpClient::with_response(responses_response("ok"));
let image = write_image(&temp, "image.png", PNG_BYTES);
let provider = "unknown";
let error = runtime(&temp, provider, 1024)
.view_image_with_http(view_args(&image), &AgentCancellation::default(), &fake)
.unwrap_err()
.to_string();
assert!(error.contains(provider), "{error}");
assert!(!error.contains("mc-secret"), "{error}");
assert!(!error.contains("openai-secret"), "{error}");
assert_eq!(fake.request_count(), 0);
let result = runtime(&temp, "openai", 1024)
.view_image_with_http(view_args(&image), &AgentCancellation::default(), &fake)
.unwrap();
assert_eq!(result.content, "ok");
let requests = fake.requests.borrow();
assert_eq!(requests[0].url, OPENAI_RESPONSES_URL);
let body = request_body_json(&requests[0]);
assert_eq!(body["input"][0]["content"][1]["type"], "input_image");
assert!(requests[0].headers["authorization"].contains("openai-secret"));
env.remove_var("MC_API_KEY");
env.remove_var("OPENAI_API_KEY");
}
#[test]
fn claude_code_request_body_and_oauth_headers_match_anthropic_messages_api() {
let config = VisionConfig {
provider: CLAUDE_CODE_PROVIDER.to_string(),
model: "sonnet".to_string(),
provider_kind: VisionProvider::ClaudeCode {
auth: ClaudeCodeAuth::OAuth(crate::providers::claude_code::auth::ClaudeOAuthCredential {
access_token: "cc-access-token".to_string(),
refresh_token: "cc-refresh-token".to_string(),
expires_at_ms: Some(4102444800000),
source: crate::providers::claude_code::auth::ClaudeCredentialSource::Keychain,
store: std::sync::Arc::new(
crate::providers::claude_code::auth::ClaudeCodeCredentialStore::with_credentials_path(
PathBuf::from("/tmp/unused"),
),
),
}),
},
text_verbosity: None,
};
let request = build_vision_request(
&config,
"system prompt",
"user prompt",
"image/png",
"QUJD".to_string(),
);
let body = request_body_json(&request);
assert_eq!(
request.url,
crate::providers::claude_code::body::messages_url()
);
assert!(request.url.ends_with("?beta=true"));
assert_eq!(request.headers["accept"], "application/json");
assert_eq!(request.headers["authorization"], "Bearer cc-access-token");
assert_eq!(request.headers["x-app"], "cli");
let beta = &request.headers["anthropic-beta"];
assert!(beta.contains("claude-code-20250219"));
assert!(beta.contains("oauth-2025-04-20"));
assert!(!request.headers.contains_key("x-api-key"));
assert_eq!(body["model"], "claude-sonnet-4-6");
assert!(
body["system"]
.as_str()
.unwrap()
.starts_with(CLAUDE_CODE_BILLING_HEADER)
);
assert!(
body["system"]
.as_str()
.unwrap()
.contains(CLAUDE_CODE_SYSTEM_PREFIX)
);
assert!(body["system"].as_str().unwrap().contains("system prompt"));
assert_eq!(body["stream"], false);
assert_eq!(body["max_tokens"], 4096);
assert_eq!(body["messages"][0]["content"][0]["type"], "image");
assert_eq!(body["messages"][0]["content"][0]["source"]["data"], "QUJD");
assert_eq!(body["messages"][0]["content"][1]["text"], "user prompt");
}
#[test]
fn claude_code_api_key_fallback_headers_use_x_api_key() {
let config = VisionConfig {
provider: CLAUDE_CODE_PROVIDER.to_string(),
model: "claude-vision".to_string(),
provider_kind: VisionProvider::ClaudeCode {
auth: ClaudeCodeAuth::ApiKey {
key: "sk-ant-api-key".to_string(),
},
},
text_verbosity: None,
};
let request = build_vision_request(
&config,
"system",
"prompt",
"image/jpeg",
"QUJD".to_string(),
);
let body = request_body_json(&request);
assert_eq!(
request.url,
crate::providers::claude_code::body::messages_url()
);
assert!(request.url.ends_with("?beta=true"));
assert!(
!body["system"]
.as_str()
.unwrap()
.contains(CLAUDE_CODE_BILLING_HEADER)
);
assert!(
body["system"]
.as_str()
.unwrap()
.starts_with(CLAUDE_CODE_SYSTEM_PREFIX)
);
assert_eq!(request.headers["x-api-key"], "sk-ant-api-key");
assert!(!request.headers.contains_key("authorization"));
assert!(!request.headers.contains_key("x-app"));
assert!(!request.headers["anthropic-beta"].contains("oauth-2025-04-20"));
}
#[test]
fn claude_code_provider_resolves_auth_and_parses_anthropic_response() {
let env = crate::test_support::env::env_lock();
env.remove_var("MC_API_KEY");
env.remove_var("OPENAI_API_KEY");
env.set_var(
"MC_CLAUDE_CODE_CREDENTIALS_PATH",
"missing-claude-code-vision-credentials.json",
);
let temp = tempfile::TempDir::new().unwrap();
let paths = crate::config::McPaths::from_root(temp.path().join("mc"));
crate::config::write_auth(
&paths,
&crate::config::Auth {
providers: BTreeMap::from([(
CLAUDE_CODE_PROVIDER.to_string(),
crate::config::AuthProviderRecord::ApiKey {
key: "claude-code-api-key".to_string(),
},
)]),
..crate::config::Auth::default()
},
)
.unwrap();
let image = write_image(&temp, "image.png", PNG_BYTES);
let mut runtime = runtime(&temp, CLAUDE_CODE_PROVIDER, 1024);
runtime.view_image_paths = Some(paths);
let fake = FakeVisionHttpClient::with_response(
json!({"content":[{"type":"text","text":"cc ok"}]}),
);
let result = runtime
.view_image_with_http(view_args(&image), &AgentCancellation::default(), &fake)
.unwrap();
let requests = fake.requests.borrow();
assert_eq!(result.content, "cc ok");
assert_eq!(
requests[0].url,
crate::providers::claude_code::body::messages_url()
);
assert_eq!(requests[0].headers["x-api-key"], "claude-code-api-key");
let body = request_body_json(&requests[0]);
assert_eq!(body["messages"][0]["content"][0]["type"], "image");
env.remove_var("MC_CLAUDE_CODE_CREDENTIALS_PATH");
}
#[test]
fn claude_code_view_image_refreshes_expired_oauth_before_request_build() {
use crate::providers::claude_code::auth::{
ClaudeCodeAuth, ClaudeCodeCredentialStore, RefreshedClaudeToken,
test_support::{ScriptedRefreshTransport, StaticKeychainReader},
};
let temp = tempfile::TempDir::new().unwrap();
let path = temp.path().join(".credentials.json");
fs::write(
&path,
r#"{"claudeAiOauth":{"accessToken":"cc-old","refreshToken":"cc-refresh","expiresAt":1}}"#,
)
.unwrap();
let calls = Arc::new(Mutex::new(0));
let store = ClaudeCodeCredentialStore::with_readers(
path,
Arc::new(StaticKeychainReader(None)),
Arc::new(ScriptedRefreshTransport {
calls: Arc::clone(&calls),
response: RefreshedClaudeToken {
access_token: "cc-new".to_string(),
refresh_token: None,
expires_at_ms: Some(4102444800000),
},
}),
);
let mut auth = ClaudeCodeAuth::OAuth(store.read_oauth_credential().unwrap().unwrap());
refresh_claude_code_view_image_auth(&mut auth, &AgentCancellation::default()).unwrap();
let config = VisionConfig {
provider: CLAUDE_CODE_PROVIDER.to_string(),
model: "sonnet".to_string(),
provider_kind: VisionProvider::ClaudeCode { auth },
text_verbosity: None,
};
let request =
build_vision_request(&config, "system", "prompt", "image/png", "QUJD".to_string());
assert_eq!(*calls.lock().unwrap(), 1);
assert_eq!(request.headers["authorization"], "Bearer cc-new");
}
#[test]
fn claude_code_view_image_fresh_oauth_skips_refresh() {
use crate::providers::claude_code::auth::{
ClaudeCodeAuth, ClaudeCodeCredentialStore, RefreshedClaudeToken,
test_support::{ScriptedRefreshTransport, StaticKeychainReader},
};
let temp = tempfile::TempDir::new().unwrap();
let path = temp.path().join(".credentials.json");
fs::write(
&path,
r#"{"claudeAiOauth":{"accessToken":"cc-fresh","refreshToken":"cc-refresh","expiresAt":4102444800000}}"#,
)
.unwrap();
let calls = Arc::new(Mutex::new(0));
let store = ClaudeCodeCredentialStore::with_readers(
path,
Arc::new(StaticKeychainReader(None)),
Arc::new(ScriptedRefreshTransport {
calls: Arc::clone(&calls),
response: RefreshedClaudeToken {
access_token: "cc-new".to_string(),
refresh_token: None,
expires_at_ms: None,
},
}),
);
let mut auth = ClaudeCodeAuth::OAuth(store.read_oauth_credential().unwrap().unwrap());
refresh_claude_code_view_image_auth(&mut auth, &AgentCancellation::default()).unwrap();
assert_eq!(*calls.lock().unwrap(), 0);
}
#[test]
fn openai_codex_view_image_refreshes_expired_oauth_before_use() {
let temp = tempfile::TempDir::new().unwrap();
let paths = crate::config::McPaths::from_root(temp.path().join("mc"));
crate::config::write_auth(
&paths,
&crate::config::Auth {
providers: BTreeMap::from([(
OPENAI_CODEX_PROVIDER.to_string(),
crate::config::AuthProviderRecord::OAuth {
access: "codex-old".to_string(),
refresh: Some("codex-refresh".to_string()),
expires: Some(1),
account_id: Some("acct_old".to_string()),
},
)]),
..crate::config::Auth::default()
},
)
.unwrap();
let calls = Arc::new(Mutex::new(0));
let calls_for_exchange = Arc::clone(&calls);
let credential =
resolve_openai_codex_view_image_credential_with_exchange(&paths, |refresh| {
*calls_for_exchange.lock().unwrap() += 1;
assert_eq!(refresh, "codex-refresh");
Ok(crate::login::NormalizedToken {
access: "codex-new".to_string(),
refresh: None,
expires: Some(4102444800),
account_id: "acct_new".to_string(),
})
})
.unwrap();
assert_eq!(*calls.lock().unwrap(), 1);
assert_eq!(
credential,
ProviderCredential::OAuth {
access: "codex-new".to_string(),
account_id: Some("acct_new".to_string())
}
);
}
#[test]
fn openai_codex_view_image_fresh_oauth_skips_refresh() {
let temp = tempfile::TempDir::new().unwrap();
let paths = crate::config::McPaths::from_root(temp.path().join("mc"));
crate::config::write_auth(
&paths,
&crate::config::Auth {
providers: BTreeMap::from([(
OPENAI_CODEX_PROVIDER.to_string(),
crate::config::AuthProviderRecord::OAuth {
access: "codex-fresh".to_string(),
refresh: Some("codex-refresh".to_string()),
expires: Some(4102444800),
account_id: Some("acct_123".to_string()),
},
)]),
..crate::config::Auth::default()
},
)
.unwrap();
let calls = Arc::new(Mutex::new(0));
let calls_for_exchange = Arc::clone(&calls);
let credential = resolve_openai_codex_view_image_credential_with_exchange(&paths, |_| {
*calls_for_exchange.lock().unwrap() += 1;
unreachable!("fresh codex token must not refresh")
})
.unwrap();
assert_eq!(*calls.lock().unwrap(), 0);
assert_eq!(
credential,
ProviderCredential::OAuth {
access: "codex-fresh".to_string(),
account_id: Some("acct_123".to_string())
}
);
}
#[test]
fn openai_codex_view_image_expired_without_refresh_reports_relogin() {
let temp = tempfile::TempDir::new().unwrap();
let paths = crate::config::McPaths::from_root(temp.path().join("mc"));
crate::config::write_auth(
&paths,
&crate::config::Auth {
providers: BTreeMap::from([(
OPENAI_CODEX_PROVIDER.to_string(),
crate::config::AuthProviderRecord::OAuth {
access: "codex-expired".to_string(),
refresh: None,
expires: Some(1),
account_id: Some("acct_123".to_string()),
},
)]),
..crate::config::Auth::default()
},
)
.unwrap();
let error = resolve_openai_codex_view_image_credential_with_exchange(&paths, |_| {
unreachable!("missing refresh token must reject before exchange")
})
.unwrap_err()
.to_string();
assert!(
error.contains("view_image provider 'openai-codex' OAuth refresh failed"),
"{error}"
);
assert!(error.contains(OPENAI_CODEX_RELOGIN_GUIDANCE), "{error}");
}
#[test]
fn openai_codex_uses_codex_backend_with_provider_keyed_oauth() {
let env = crate::test_support::env::env_lock();
env.remove_var("MC_API_KEY");
env.remove_var("OPENAI_API_KEY");
let temp = tempfile::TempDir::new().unwrap();
let paths = crate::config::McPaths::from_root(temp.path().join("mc"));
crate::config::write_auth(
&paths,
&crate::config::Auth {
providers: BTreeMap::from([(
OPENAI_CODEX_PROVIDER.to_string(),
crate::config::AuthProviderRecord::OAuth {
access: "codex-access".to_string(),
refresh: Some("codex-refresh".to_string()),
expires: Some(4102444800),
account_id: Some("acct_123".to_string()),
},
)]),
..crate::config::Auth::default()
},
)
.unwrap();
let image = write_image(&temp, "image.png", PNG_BYTES);
let mut runtime = runtime(&temp, OPENAI_CODEX_PROVIDER, 1024);
runtime.view_image_paths = Some(paths);
let fake = FakeVisionHttpClient::with_text(codex_sse_response("codex ok"));
let result = runtime
.view_image_with_http(view_args(&image), &AgentCancellation::default(), &fake)
.unwrap();
let requests = fake.requests.borrow();
let request = &requests[0];
let body = request_body_json(request);
assert_eq!(result.content, "codex ok");
assert_eq!(request.url, CODEX_RESPONSES_URL);
assert_ne!(request.url, OPENAI_RESPONSES_URL);
assert_eq!(request.headers["authorization"], "Bearer codex-access");
assert_eq!(request.headers["chatgpt-account-id"], "acct_123");
assert_eq!(request.headers["originator"], "codex_cli_rs");
assert_eq!(request.headers["user-agent"], "codex_cli_rs/0.144.0");
assert_eq!(request.headers["openai-beta"], "responses=experimental");
assert_eq!(request.headers["accept"], "text/event-stream");
assert_eq!(request.headers["content-type"], "application/json");
assert_eq!(body["model"], "vision-model");
assert_eq!(body["store"], false);
assert_eq!(body["stream"], true);
assert_eq!(body["instructions"], SYSTEM_PROMPT.trim());
assert_eq!(body["text"]["verbosity"], "low");
assert_eq!(body["include"][0], "reasoning.encrypted_content");
assert_eq!(body["input"][0]["role"], "user");
assert_eq!(body["input"][0]["content"][0]["type"], "input_text");
assert_eq!(body["input"][0]["content"][0]["text"], "describe image");
assert_eq!(body["input"][0]["content"][1]["type"], "input_image");
assert_eq!(
body["input"][0]["content"][1]["image_url"],
"data:image/png;base64,iVBORw0KGgo="
);
assert!(body.get("tools").is_none());
assert!(body.get("tool_choice").is_none());
assert!(body.get("parallel_tool_calls").is_none());
}
#[test]
fn openai_codex_resolves_account_id_from_jwt_and_rejects_missing_account_before_http() {
let env = crate::test_support::env::env_lock();
env.remove_var("MC_API_KEY");
env.remove_var("OPENAI_API_KEY");
let temp = tempfile::TempDir::new().unwrap();
let paths = crate::config::McPaths::from_root(temp.path().join("mc"));
crate::config::write_auth(
&paths,
&crate::config::Auth {
providers: BTreeMap::from([(
OPENAI_CODEX_PROVIDER.to_string(),
crate::config::AuthProviderRecord::OAuth {
access: fake_jwt("acct_jwt"),
refresh: Some("codex-refresh".to_string()),
expires: Some(4102444800),
account_id: None,
},
)]),
..crate::config::Auth::default()
},
)
.unwrap();
let image = write_image(&temp, "image.png", PNG_BYTES);
let mut runtime = runtime(&temp, OPENAI_CODEX_PROVIDER, 1024);
runtime.view_image_paths = Some(paths.clone());
let fake = FakeVisionHttpClient::with_text(codex_sse_response("jwt ok"));
runtime
.view_image_with_http(view_args(&image), &AgentCancellation::default(), &fake)
.unwrap();
assert_eq!(
fake.requests.borrow()[0].headers["chatgpt-account-id"],
"acct_jwt"
);
crate::config::write_auth(
&paths,
&crate::config::Auth {
providers: BTreeMap::from([(
OPENAI_CODEX_PROVIDER.to_string(),
crate::config::AuthProviderRecord::OAuth {
access: "not-a-jwt".to_string(),
refresh: Some("codex-refresh".to_string()),
expires: Some(4102444800),
account_id: None,
},
)]),
..crate::config::Auth::default()
},
)
.unwrap();
let fake = FakeVisionHttpClient::with_text(codex_sse_response("unused"));
let error = runtime
.view_image_with_http(view_args(&image), &AgentCancellation::default(), &fake)
.unwrap_err()
.to_string();
assert!(error.contains("missing ChatGPT account id"), "{error}");
assert_eq!(fake.request_count(), 0);
}
#[test]
fn successful_provider_echoed_data_url_is_redacted_from_all_result_surfaces() {
let temp = tempfile::TempDir::new().unwrap();
let image = write_image(&temp, "image.png", PNG_BYTES);
let runtime = runtime(&temp, "local-vision", 1024);
let echoed = "provider echoed data:image/png;base64,YWJj in success response";
let fake = FakeVisionHttpClient::with_response(openai_response(echoed));
let result = runtime
.view_image_with_http(view_args(&image), &AgentCancellation::default(), &fake)
.unwrap();
let call = crate::providers::ToolCall {
id: "call_view_image".to_string(),
name: tool_name::VIEW_IMAGE.to_string(),
arguments: json!({"path": image.display().to_string(), "prompt": "describe image"}),
};
let display_summary = crate::output::tool_display_summary(&call, &result);
let hook_like_payload = json!({"tool":"view_image","result":result});
let surfaces = format!(
"{}{}{}{:?}{}",
hook_like_payload["result"]["content"],
hook_like_payload["result"]["metadata"],
serde_json::to_string(&hook_like_payload["result"]).unwrap(),
display_summary,
hook_like_payload
);
assert!(
surfaces.contains("data:<redacted>;base64,<redacted>"),
"{surfaces}"
);
assert!(!surfaces.contains("data:image"), "{surfaces}");
assert!(!surfaces.contains("YWJj"), "{surfaces}");
assert!(!surfaces.contains("messages"), "{surfaces}");
}
#[test]
fn serialized_result_display_and_hook_like_payloads_do_not_contain_image_bytes() {
let temp = tempfile::TempDir::new().unwrap();
let image = write_image(&temp, "image.png", PNG_BYTES);
let runtime = runtime(&temp, "local-vision", 1024);
let fake = FakeVisionHttpClient::with_response(openai_response("answer"));
let result = runtime
.view_image_with_http(view_args(&image), &AgentCancellation::default(), &fake)
.unwrap();
let hook_like_payload = json!({"tool":"view_image","result":result});
let text = hook_like_payload.to_string();
assert!(!text.contains("data:image"), "{text}");
assert!(!text.contains("iVBORw0KGgo="), "{text}");
assert!(!text.contains("messages"), "{text}");
}
#[test]
fn provider_error_echoing_base64_is_bounded_and_redacted() {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let url = format!(
"http://{}/v1/chat/completions",
listener.local_addr().unwrap()
);
let echoed = format!("data:image/png;base64,{}", "A".repeat(6000));
let server = std::thread::spawn(move || {
let (mut stream, _) = listener.accept().unwrap();
let mut buffer = [0_u8; 8192];
let _ = stream.read(&mut buffer).unwrap();
let response = format!(
"HTTP/1.1 500 Internal Server Error\r\ncontent-length: {}\r\n\r\n{}",
echoed.len(),
echoed
);
stream.write_all(response.as_bytes()).unwrap();
});
let error = post_json(
VisionHttpRequest {
url,
headers: json_headers(),
body: json!({"x":"y"}).to_string(),
},
&AgentCancellation::default(),
)
.unwrap_err()
.to_string();
server.join().unwrap();
assert!(!error.contains("data:image/png;base64"), "{error}");
assert!(!error.contains(&"A".repeat(256)), "{error}");
assert!(error.len() < 4600, "{}", error.len());
}
#[test]
fn post_json_observes_cancellation_during_header_wait() {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let url = format!(
"http://{}/v1/chat/completions",
listener.local_addr().unwrap()
);
let (accepted_tx, accepted_rx) = mpsc::channel();
let server = std::thread::spawn(move || {
let (mut stream, _) = listener.accept().unwrap();
accepted_tx.send(()).unwrap();
let mut buffer = [0_u8; 8192];
let _ = stream.read(&mut buffer);
});
let (cancellation, handle): (AgentCancellation, AgentCancellationHandle) =
AgentCancellation::default().child_token();
let canceler = std::thread::spawn(move || {
accepted_rx.recv().unwrap();
std::thread::sleep(Duration::from_millis(50));
handle.cancel();
});
let started = Instant::now();
let _error = post_json(
VisionHttpRequest {
url,
headers: json_headers(),
body: json!({"x":"y"}).to_string(),
},
&cancellation,
)
.unwrap_err();
canceler.join().unwrap();
server.join().unwrap();
assert!(started.elapsed() < Duration::from_secs(1));
}
#[test]
fn response_cap_sets_truncated_metadata() {
let temp = tempfile::TempDir::new().unwrap();
let image = write_image(&temp, "image.png", PNG_BYTES);
let runtime = runtime(&temp, "local-vision", 1024);
let fake = FakeVisionHttpClient::with_response(openai_response(&"a".repeat(70 * 1024)));
let result = runtime
.view_image_with_http(view_args(&image), &AgentCancellation::default(), &fake)
.unwrap();
assert_eq!(result.metadata[meta::TRUNCATED], true);
assert!(result.content.len() <= VIEW_IMAGE_RESPONSE_MAX_BYTES);
assert!(
result
.content
.contains("[view_image response truncated at 65536 bytes]")
);
}
}