use anyhow::Result;
use async_trait::async_trait;
#[cfg(feature = "gateway")]
use std::collections::BTreeMap;
#[cfg(feature = "gateway")]
use std::sync::Arc;
#[async_trait]
pub trait VisionAdapterTrait: Send + Sync {
async fn send_vision_request(
&self,
screenshot_b64: &str,
prompt: &str,
system_prompt: &str,
model: &str,
) -> Result<String>;
}
#[cfg(feature = "gateway")]
pub struct FluxweirVisionAdapter {
client: Option<Arc<fluxweir::sdk::FluxweirClient>>,
}
#[cfg(feature = "gateway")]
impl FluxweirVisionAdapter {
pub fn new(client: Option<Arc<fluxweir::sdk::FluxweirClient>>) -> Self {
Self { client }
}
}
#[cfg(feature = "gateway")]
#[async_trait]
impl VisionAdapterTrait for FluxweirVisionAdapter {
async fn send_vision_request(
&self,
screenshot_b64: &str,
prompt: &str,
system_prompt: &str,
model: &str,
) -> Result<String> {
let Some(client) = &self.client else {
anyhow::bail!(
"vision request requires the fluxweir gateway: set [fluxweir].enabled=true \
(mode embedded/external) — engine-mllm carries vision via the gateway"
);
};
let model_name = model.rsplit(':').next().unwrap_or(model);
let image_part = fluxweir::sdk::ChatContentPart::ImageUrl {
image_url: fluxweir::sdk::ImageUrlBody {
url: format!("data:image/jpeg;base64,{screenshot_b64}"),
},
};
let text_part = fluxweir::sdk::ChatContentPart::text(prompt);
let user_message =
fluxweir::sdk::ChatMessage::user(fluxweir::sdk::ChatContent::Parts(vec![
image_part, text_part,
]));
let system_message = fluxweir::sdk::ChatMessage::system(system_prompt.to_string());
let chat_req = fluxweir::sdk::ChatRequest {
model: model_name.to_string(),
messages: vec![system_message, user_message],
config: fluxweir::sdk::RequestConfig::default(),
tools: Vec::new(),
tool_choice: None,
vendor_extensions: BTreeMap::new(),
};
let response = client
.chat(&chat_req)
.await
.map_err(|e| anyhow::anyhow!("Vision LLM call failed for model {}: {:?}", model, e))?;
if response.content.trim().is_empty() {
anyhow::bail!("Vision LLM returned empty content");
}
Ok(response.content)
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::atomic::{AtomicU32, Ordering};
use std::sync::Arc;
#[cfg(feature = "gateway")]
#[tokio::test]
async fn disabled_gateway_returns_structured_error() {
let adapter = FluxweirVisionAdapter::new(None);
let err = adapter
.send_vision_request("b64", "prompt", "system", "openai:gw-model")
.await
.expect_err("must error without client");
let rendered = format!("{err:#}");
assert!(rendered.contains("[fluxweir].enabled=true"), "{rendered}");
assert!(rendered.contains("fluxweir gateway"), "{rendered}");
}
#[test]
fn model_name_strips_adapter_prefix() {
let prefixed = "openai:gw-model";
assert_eq!(prefixed.rsplit(':').next().unwrap_or(prefixed), "gw-model");
let plain = "qwen3-vl";
assert_eq!(plain.rsplit(':').next().unwrap_or(plain), "qwen3-vl");
let namespaced = "vendor:family:model";
assert_eq!(namespaced.rsplit(':').next().unwrap_or(namespaced), "model");
}
struct MockVisionAdapter {
response: String,
call_count: Arc<AtomicU32>,
}
impl MockVisionAdapter {
fn new(response: &str) -> (Self, Arc<AtomicU32>) {
let count = Arc::new(AtomicU32::new(0));
(
Self {
response: response.to_string(),
call_count: count.clone(),
},
count,
)
}
}
#[async_trait]
impl VisionAdapterTrait for MockVisionAdapter {
async fn send_vision_request(
&self,
_screenshot_b64: &str,
_prompt: &str,
_system_prompt: &str,
_model: &str,
) -> Result<String> {
self.call_count.fetch_add(1, Ordering::SeqCst);
Ok(self.response.clone())
}
}
#[tokio::test]
async fn test_mock_vision_adapter_returns_response() {
let json_response = r##"{"action": "click", "selector": "#btn", "reasoning": "test"}"##;
let (adapter, count) = MockVisionAdapter::new(json_response);
let result = adapter
.send_vision_request("base64data", "analyze", "system", "test-model")
.await
.unwrap();
assert_eq!(result, json_response);
assert_eq!(count.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn test_mock_vision_adapter_call_count() {
let (adapter, count) = MockVisionAdapter::new(r#"{"action": "done"}"#);
for _ in 0..3 {
let _ = adapter
.send_vision_request("img", "prompt", "sys", "model")
.await;
}
assert_eq!(count.load(Ordering::SeqCst), 3);
}
#[cfg(feature = "gateway")]
#[test]
fn test_fluxweir_vision_adapter_construction() {
let disabled = FluxweirVisionAdapter::new(None);
let _ = disabled;
}
#[tokio::test]
async fn test_vision_adapter_trait_object() {
let (adapter, _) = MockVisionAdapter::new(r#"{"action": "extract"}"#);
let boxed: Box<dyn VisionAdapterTrait> = Box::new(adapter);
let result = boxed
.send_vision_request("img", "prompt", "sys", "model")
.await
.unwrap();
assert!(result.contains("extract"));
}
}