use std::time::Duration;
use serde::{Deserialize, Serialize};
pub const VISION_MODEL_PATTERNS: &[&str] = &[
"qwen3-vl",
"llava",
"bakllava",
"qwen2-vl",
"gemma3-vl",
"pixtral",
"minicpm-v",
"moondream",
];
pub const DEFAULT_VLM_AUTO_PULL: &str = "qwen3-vl:8b-thinking";
pub const DEFAULT_WHISPER_MODEL: &str = "base";
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct Capabilities {
pub vlm: Option<VlmBackend>,
pub stt: Option<SttBackend>,
pub ocr: Option<OcrBackend>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct VlmBackend {
pub endpoint: String,
pub model: String,
pub source: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SttBackend {
pub kind: SttKind,
pub endpoint: String,
pub model: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum SttKind {
WhisperCli,
OpenAIApi,
LocalServer,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct OcrBackend {
pub binary: String,
}
pub async fn probe() -> Capabilities {
let (vlm, stt, ocr) = tokio::join!(probe_vlm(), probe_stt(), probe_ocr());
Capabilities { vlm, stt, ocr }
}
async fn probe_vlm() -> Option<VlmBackend> {
let client = match crate::http_client::timed_client(Duration::from_millis(800)) {
Ok(c) => c,
Err(_) => return None,
};
let ollama_endpoint = "http://localhost:11434";
if let Some(backend) = probe_vlm_ollama(&client, ollama_endpoint).await {
return Some(backend);
}
let llama_endpoint = std::env::var("LLAMACPP_VLM_ENDPOINT")
.unwrap_or_else(|_| "http://localhost:8000".to_string());
if let Some(backend) = probe_vlm_llamacpp(&client, &llama_endpoint).await {
return Some(backend);
}
None
}
async fn probe_vlm_ollama(client: &reqwest::Client, endpoint: &str) -> Option<VlmBackend> {
let resp = client
.get(format!("{endpoint}/api/tags"))
.send()
.await
.ok()?;
if !resp.status().is_success() {
return None;
}
#[derive(Deserialize)]
struct OllamaTags {
models: Vec<OllamaModel>,
}
#[derive(Deserialize)]
struct OllamaModel {
name: String,
}
let body: OllamaTags = resp.json().await.ok()?;
let model = pick_best_vision_model(
&body
.models
.iter()
.map(|m| m.name.as_str())
.collect::<Vec<_>>(),
)?;
Some(VlmBackend {
endpoint: endpoint.to_string(),
model: model.to_string(),
source: "ollama".to_string(),
})
}
async fn probe_vlm_llamacpp(client: &reqwest::Client, endpoint: &str) -> Option<VlmBackend> {
let resp = client
.get(format!("{endpoint}/v1/models"))
.send()
.await
.ok()?;
if !resp.status().is_success() {
return None;
}
#[derive(Deserialize)]
struct LlamaModels {
data: Vec<LlamaModel>,
}
#[derive(Deserialize)]
struct LlamaModel {
id: String,
}
let body: LlamaModels = resp.json().await.ok()?;
let model = body.data.first()?;
Some(VlmBackend {
endpoint: endpoint.to_string(),
model: model.id.clone(),
source: "llamacpp".to_string(),
})
}
pub fn pick_best_vision_model<'a>(model_names: &[&'a str]) -> Option<&'a str> {
for pattern in VISION_MODEL_PATTERNS {
for name in model_names {
if name.contains(pattern) {
return Some(name);
}
}
}
None
}
async fn probe_stt() -> Option<SttBackend> {
if let Some(path) = which("whisper").or_else(|| which("whisper-cli")) {
return Some(SttBackend {
kind: SttKind::WhisperCli,
endpoint: path,
model: Some(DEFAULT_WHISPER_MODEL.to_string()),
});
}
if std::env::var("OPENAI_API_KEY").is_ok() {
return Some(SttBackend {
kind: SttKind::OpenAIApi,
endpoint: "https://api.openai.com/v1/audio/transcriptions".to_string(),
model: Some("whisper-1".to_string()),
});
}
let client = crate::http_client::timed_client(Duration::from_millis(300)).ok()?;
if client
.get("http://localhost:9000/")
.send()
.await
.map(|r| r.status().is_success() || r.status().as_u16() == 405)
.unwrap_or(false)
{
return Some(SttBackend {
kind: SttKind::LocalServer,
endpoint: "http://localhost:9000/asr".to_string(),
model: None,
});
}
None
}
async fn probe_ocr() -> Option<OcrBackend> {
which("tesseract").map(|binary| OcrBackend { binary })
}
pub fn which(binary: &str) -> Option<String> {
let path = std::env::var_os("PATH")?;
for dir in std::env::split_paths(&path) {
let candidate = dir.join(binary);
if candidate.is_file() {
return Some(candidate.to_string_lossy().into_owned());
}
}
None
}
impl Capabilities {
pub fn any(&self) -> bool {
self.vlm.is_some() || self.stt.is_some() || self.ocr.is_some()
}
pub fn summary(&self) -> String {
let mut parts = Vec::new();
if let Some(v) = &self.vlm {
parts.push(format!("vlm:{} ({})", v.model, v.source));
}
if let Some(s) = &self.stt {
parts.push(format!("stt:{:?}", s.kind));
}
if self.ocr.is_some() {
parts.push("ocr:tesseract".to_string());
}
if parts.is_empty() {
"no backends detected".to_string()
} else {
parts.join(", ")
}
}
pub fn install_hints(&self) -> Vec<String> {
let mut out = Vec::new();
if self.vlm.is_none() {
out.push(
"vlm: install ollama (https://ollama.com/download) and run \
`ollama pull qwen3-vl:8b-thinking`: captchaforge will then \
auto-detect it. Solves canvas / SVG / image-grid CAPTCHAs."
.to_string(),
);
}
if self.stt.is_none() {
out.push(
"stt: install OpenAI Whisper CLI (`pip install -U openai-whisper`) \
OR set OPENAI_API_KEY for the hosted Whisper API. Solves \
reCAPTCHA v2 audio + generic spoken-digit CAPTCHAs."
.to_string(),
);
}
if self.ocr.is_none() {
out.push(
"ocr: install tesseract (`apt install tesseract-ocr` / \
`brew install tesseract`). Used as a fallback when no VLM \
is available."
.to_string(),
);
}
out
}
pub async fn ensure_vlm_model(&mut self) -> anyhow::Result<bool> {
if self.vlm.is_some() {
return Ok(true);
}
let ollama = which("ollama");
let Some(ollama_bin) = ollama else {
return Ok(false);
};
let client = crate::http_client::timed_client(Duration::from_millis(800))?;
if client
.get("http://localhost:11434/api/tags")
.send()
.await
.map(|r| !r.status().is_success())
.unwrap_or(true)
{
return Ok(false);
}
tracing::info!(
model = DEFAULT_VLM_AUTO_PULL,
"auto-pulling vision model via Ollama (first-run install)"
);
let status = std::process::Command::new(&ollama_bin)
.arg("pull")
.arg(DEFAULT_VLM_AUTO_PULL)
.status()?;
if !status.success() {
anyhow::bail!("ollama pull {DEFAULT_VLM_AUTO_PULL} exited {status}");
}
self.vlm = probe_vlm().await;
Ok(self.vlm.is_some())
}
}
#[cfg(test)]
#[path = "backends/tests.rs"]
mod tests;