use serde_json::Value;
pub const LOCAL_DEFAULT_ENDPOINTS: &[(&str, &str)] = &[
("ollama", "http://localhost:11434"),
("lmstudio", "http://localhost:1234"),
];
pub const KEYED_CLOUD_PROVIDERS: &[&str] = &["openai", "anthropic", "openrouter", "portkey"];
#[derive(Clone, Debug, PartialEq)]
pub enum ProbeKind {
ListModels {
target: String,
reach: Option<HostPort>,
},
RouteCheck {
capability: String,
provider: String,
model: String,
reach: Option<HostPort>,
},
Generate {
provider: Option<String>,
model: Option<String>,
},
}
#[derive(Clone, Debug, PartialEq)]
pub struct ProbeSpec {
pub label: String,
pub kind: ProbeKind,
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum Verdict {
Proven,
NotProven,
Failed,
}
impl Verdict {
pub fn glyph(&self) -> &'static str {
match self {
Verdict::Proven => "✓",
Verdict::NotProven => "?",
Verdict::Failed => "✗",
}
}
pub fn word(&self) -> &'static str {
match self {
Verdict::Proven => "proven",
Verdict::NotProven => "NOT PROVEN",
Verdict::Failed => "FAILED",
}
}
}
#[derive(Clone, Debug)]
pub struct TestResult {
pub when: String,
pub label: String,
pub verdict: Verdict,
pub detail: String,
}
#[derive(Clone, Debug, PartialEq)]
pub struct HostPort {
pub host: String,
pub port: u16,
}
pub fn parse_http_host_port(url: &str) -> Option<HostPort> {
let rest = url.trim().strip_prefix("http://")?;
let authority = rest.split('/').next()?;
if authority.is_empty() || authority.contains('@') {
return None;
}
let (host, port) = match authority.rsplit_once(':') {
Some((h, p)) => (h, p.parse::<u16>().ok()?),
None => (authority, 80),
};
if host.is_empty() {
return None;
}
Some(HostPort {
host: host.to_string(),
port,
})
}
pub fn endpoint_for(target: &str, profile_base_url: Option<&str>) -> Option<HostPort> {
if let Some(url) = profile_base_url {
return parse_http_host_port(url);
}
LOCAL_DEFAULT_ENDPOINTS
.iter()
.find(|(p, _)| *p == target)
.and_then(|(_, url)| parse_http_host_port(url))
}
#[derive(Clone, Debug, PartialEq)]
pub enum Reach {
Connected,
Refused(String),
Unresolvable(String),
}
impl Reach {
pub fn describe(&self, hp: &HostPort) -> String {
match self {
Reach::Connected => format!(
"TCP {}:{} accepts connections — the server is UP but reports zero models",
hp.host, hp.port
),
Reach::Refused(e) => {
format!("TCP {}:{} → {e} — the server looks DOWN", hp.host, hp.port)
}
Reach::Unresolvable(e) => format!("cannot resolve {}:{} — {e}", hp.host, hp.port),
}
}
}
pub fn fold_list_models(v: &Value, reach: Option<(&HostPort, &Reach)>) -> (Verdict, String) {
let errors: Vec<String> = v
.get("errors")
.and_then(Value::as_array)
.map(|a| {
a.iter()
.map(|e| match e.as_str() {
Some(s) => s.to_string(),
None => e.to_string(),
})
.collect()
})
.unwrap_or_default();
if let Some(first) = errors.first() {
return (Verdict::Failed, first.clone());
}
let models: Vec<&str> = v
.get("models")
.and_then(Value::as_array)
.map(|a| a.iter().filter_map(Value::as_str).collect())
.unwrap_or_default();
if !models.is_empty() {
return (
Verdict::Proven,
format!("{} models available · e.g. {}", models.len(), models[0]),
);
}
let base = "the CLI reports success with ZERO models (also its answer for a dead server)";
match reach {
Some((hp, r)) => (Verdict::NotProven, format!("{}; {base}", r.describe(hp))),
None => (
Verdict::NotProven,
format!("{base}; no known endpoint to reach-check"),
),
}
}
pub fn fold_route_check(
v: &Value,
model: &str,
reach: Option<(&HostPort, &Reach)>,
) -> (Verdict, String) {
let (list_verdict, list_detail) = fold_list_models(v, reach);
if list_verdict != Verdict::Proven {
return (list_verdict, list_detail);
}
let models: Vec<&str> = v
.get("models")
.and_then(Value::as_array)
.map(|a| a.iter().filter_map(Value::as_str).collect())
.unwrap_or_default();
if models.contains(&model) {
(
Verdict::Proven,
format!("model {model} is among the {} served", models.len()),
)
} else {
(
Verdict::Failed,
format!(
"model {model} is NOT among the {} the provider serves — edit the route (e) and pick from the live list",
models.len()
),
)
}
}
pub fn fold_generation(stdout: &str, elapsed_s: u64) -> (Verdict, String) {
if let Some(l) = stdout
.lines()
.find(|l| l.contains("❌") || l.trim_start().starts_with("Error:"))
{
return (Verdict::Failed, l.trim().to_string());
}
let reply: String = stdout
.lines()
.filter(|l| !is_log_line(l))
.collect::<Vec<_>>()
.join(" ")
.trim()
.to_string();
if reply.is_empty() {
return (
Verdict::NotProven,
"exit 0 but no reply text — nothing to show for it".into(),
);
}
let head: String = reply.chars().take(60).collect();
let ellipsis = if reply.chars().count() > 60 {
"…"
} else {
""
};
(
Verdict::Proven,
format!("replied in {elapsed_s}s: “{head}{ellipsis}”"),
)
}
fn is_log_line(l: &str) -> bool {
let t = l.trim_start();
t.len() > 10
&& t.as_bytes()[2] == b':'
&& t.as_bytes()[5] == b':'
&& t.get(8..).is_some_and(|r| r.trim_start().starts_with('['))
}
pub fn list_models(target: &str, profile_base_url: Option<&str>) -> ProbeSpec {
ProbeSpec {
label: format!("test {target}"),
kind: ProbeKind::ListModels {
target: target.to_string(),
reach: endpoint_for(target, profile_base_url),
},
}
}
pub fn route_check(
capability: &str,
provider: &str,
model: &str,
profile_base_url: Option<&str>,
) -> ProbeSpec {
ProbeSpec {
label: format!("test route {capability}"),
kind: ProbeKind::RouteCheck {
capability: capability.to_string(),
provider: provider.to_string(),
model: model.to_string(),
reach: endpoint_for(provider, profile_base_url),
},
}
}
pub fn generate_default() -> ProbeSpec {
ProbeSpec {
label: "generation test (default route)".into(),
kind: ProbeKind::Generate {
provider: None,
model: None,
},
}
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[test]
fn url_parse_is_http_only_and_exact() {
assert_eq!(
parse_http_host_port("http://localhost:11434"),
Some(HostPort {
host: "localhost".into(),
port: 11434
})
);
assert_eq!(
parse_http_host_port("http://h/v1/models"),
Some(HostPort {
host: "h".into(),
port: 80
})
);
assert_eq!(parse_http_host_port("https://api.openai.com/v1"), None);
assert_eq!(parse_http_host_port("http://"), None);
assert_eq!(parse_http_host_port("http://h:notaport"), None);
assert_eq!(parse_http_host_port(""), None);
assert_eq!(parse_http_host_port("http://user:pass@h:1234"), None);
assert_eq!(
parse_http_host_port("http://[::1]:1234").map(|hp| hp.host),
Some("[::1]".into())
);
}
#[test]
fn endpoints_config_first_then_local_defaults() {
let hp = endpoint_for("ollama", Some("http://10.0.0.5:11434")).unwrap();
assert_eq!(hp.host, "10.0.0.5");
assert_eq!(endpoint_for("lmstudio", None).unwrap().port, 1234);
assert_eq!(endpoint_for("openai", None), None);
assert_eq!(endpoint_for("vllm", None), None);
}
#[test]
fn zero_count_success_is_not_proven() {
let dead = json!({"ok": true, "count": 0, "errors": [], "models": []});
let (v, d) = fold_list_models(&dead, None);
assert_eq!(v, Verdict::NotProven);
assert!(d.contains("ZERO models"), "{d}");
let hp = HostPort {
host: "localhost".into(),
port: 11434,
};
let (v, d) = fold_list_models(
&dead,
Some((&hp, &Reach::Refused("connection refused".into()))),
);
assert_eq!(v, Verdict::NotProven);
assert!(d.contains("looks DOWN"), "{d}");
let (v, d) = fold_list_models(&dead, Some((&hp, &Reach::Connected)));
assert_eq!(v, Verdict::NotProven);
assert!(d.contains("UP but reports zero"), "{d}");
}
#[test]
fn listed_models_prove_and_errors_fail() {
let live = json!({"ok": true, "count": 2, "errors": [], "models": ["a", "b"]});
let (v, d) = fold_list_models(&live, None);
assert_eq!(v, Verdict::Proven);
assert!(d.contains("2 models available"), "{d}");
let broken = json!({"ok": false, "count": 0, "errors": ["401 unauthorized"], "models": []});
let (v, d) = fold_list_models(&broken, None);
assert_eq!(v, Verdict::Failed);
assert_eq!(d, "401 unauthorized");
let drifted = json!({"ok": true, "count": 9, "errors": [], "models": []});
let (v, _) = fold_list_models(&drifted, None);
assert_eq!(v, Verdict::NotProven);
}
#[test]
fn route_check_label_is_pair_free() {
let a = route_check("input.text", "lmstudio", "m1", None);
let b = route_check("input.text", "ollama", "m2", None);
assert_eq!(a.label, b.label);
assert_eq!(a.label, "test route input.text");
}
#[test]
fn route_check_judges_membership() {
let live = json!({"ok": true, "count": 2, "errors": [], "models": ["m1", "m2"]});
let (v, _) = fold_route_check(&live, "m1", None);
assert_eq!(v, Verdict::Proven);
let (v, d) = fold_route_check(&live, "ghost", None);
assert_eq!(v, Verdict::Failed);
assert!(d.contains("ghost"), "{d}");
let dead = json!({"ok": true, "count": 0, "errors": [], "models": []});
let (v, _) = fold_route_check(&dead, "m1", None);
assert_eq!(v, Verdict::NotProven);
}
#[test]
fn generation_folds_reply_error_and_silence() {
let (v, d) = fold_generation("❌ Error: API error: [Errno 61] Connection refused", 1);
assert_eq!(v, Verdict::Failed);
assert!(d.contains("Connection refused"), "{d}");
let (v, d) = fold_generation("22:31:13 [ERROR] OllamaProvider: boom\nOK\n", 3);
assert_eq!(v, Verdict::Proven);
assert!(
d.contains("OK") && !d.contains("boom"),
"log lines are noise: {d}"
);
let (v, _) = fold_generation(" \n", 2);
assert_eq!(v, Verdict::NotProven);
let (v, d) = fold_generation("ab:cd:xé[test] whatever", 1);
assert_eq!(v, Verdict::Proven, "{d}");
let long = "é".repeat(200);
let (v, d) = fold_generation(&long, 9);
assert_eq!(v, Verdict::Proven);
assert!(d.contains('…'), "{d}");
}
}