Skip to main content

byob_protocol/
connect.rs

1//! Connecting byob to an inference endpoint: the state of the link, why it fails (in words a
2//! user can act on), a connection test, local servers found at startup, and presets for
3//! common setups.
4
5use serde::{Deserialize, Serialize};
6
7/// Longest base URL a client may send.
8pub const MAX_URL_CHARS: usize = 2048;
9/// Longest model name a client may send.
10pub const MAX_MODEL_CHARS: usize = 256;
11/// Longest API key a client may send.
12pub const MAX_KEY_CHARS: usize = 4096;
13
14/// What went wrong talking to the inference endpoint.
15#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
16#[serde(rename_all = "snake_case")]
17pub enum Problem {
18    /// The URL isn't an http(s) URL.
19    BadUrl,
20    /// Nothing answered: connection refused, unknown host, timeout.
21    Unreachable,
22    /// HTTP 401/403: missing or wrong API key.
23    Unauthorized,
24    /// HTTP 404 for the API itself: usually a base URL without (or with an extra) `/v1`.
25    WrongPath,
26    /// The server doesn't have the model that was asked for.
27    ModelNotFound,
28    /// HTTP 402/429: rate limit or out of credits.
29    RateLimited,
30    /// HTTP 5xx: the server is there but failing (often still loading the model).
31    ServerError,
32}
33
34impl Problem {
35    /// Recognise a connection problem in an error message from the engine (the text of
36    /// `byob_llm::LlmError`, which entries and test results carry). `None` for anything
37    /// else (tool loops, cancelled turns, …).
38    pub fn classify(error: &str) -> Option<Self> {
39        let e = error.trim();
40        if e.starts_with("invalid inference URL") || e.starts_with("inference URL is invalid") {
41            return Some(Self::BadUrl);
42        }
43        if e.starts_with("could not reach ") {
44            return Some(Self::Unreachable);
45        }
46        let rest = e.strip_prefix("server returned HTTP ")?;
47        let (code, message) = rest.split_once(':').unwrap_or((rest, ""));
48        let status: u16 = code.trim().parse().ok()?;
49        let m = message.to_ascii_lowercase();
50        let about_model = m.contains("model")
51            && ["not found", "does not exist", "unknown", "not exist", "no such", "invalid model", "not available", "pull"].iter().any(|w| m.contains(w));
52        Some(match status {
53            401 | 403 => Self::Unauthorized,
54            404 if m.contains("model") => Self::ModelNotFound,
55            400 | 422 if about_model => Self::ModelNotFound,
56            404 | 405 => Self::WrongPath,
57            402 | 429 => Self::RateLimited,
58            500..=599 => Self::ServerError,
59            _ => return None,
60        })
61    }
62
63    /// A short headline, e.g. "Can't reach the model server".
64    pub fn title(self) -> &'static str {
65        match self {
66            Self::BadUrl => "That isn't a valid server URL",
67            Self::Unreachable => "Can't reach the model server",
68            Self::Unauthorized => "The server refused the API key",
69            Self::WrongPath => "The server is there, but not at that path",
70            Self::ModelNotFound => "The server doesn't have that model",
71            Self::RateLimited => "Rate-limited or out of credits",
72            Self::ServerError => "The model server reported an error",
73        }
74    }
75
76    /// What to do about it, in a sentence or two.
77    pub fn advice(self) -> &'static str {
78        match self {
79            Self::BadUrl => "Use the server's OpenAI-compatible base URL, like http://127.0.0.1:8080/v1.",
80            Self::Unreachable => "Nothing answered at that address. Start your server (llama-server, Ollama, LM Studio, vLLM…) or check the URL and port.",
81            Self::Unauthorized => "Hosted APIs need a key. Enter or replace the API key in the connection settings.",
82            Self::WrongPath => "OpenAI-compatible APIs usually end in /v1. Check the base URL.",
83            Self::ModelNotFound => "Pick one of the models the server lists, or load the model on the server first.",
84            Self::RateLimited => "The provider is limiting requests or your account needs credits. Wait a moment, or check your plan.",
85            Self::ServerError => "The server may still be loading the model. Retry in a moment, or check the server's log.",
86        }
87    }
88}
89
90/// The engine's view of its current endpoint.
91#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
92#[serde(tag = "state", rename_all = "snake_case")]
93pub enum Link {
94    /// Not checked (yet).
95    #[default]
96    Unknown,
97    /// A check is running.
98    Checking,
99    /// The endpoint answered (a check or a reply worked).
100    Ok,
101    /// The last check or reply failed to reach it. `detail` is the raw error.
102    Failed {
103        #[serde(default)]
104        problem: Option<Problem>,
105        #[serde(default)]
106        detail: String,
107    },
108}
109
110impl Link {
111    pub fn is_ok(&self) -> bool {
112        *self == Self::Ok
113    }
114
115    /// The failure, as an error from a check or a reply.
116    pub fn failed(detail: impl Into<String>) -> Self {
117        let detail = detail.into();
118        Self::Failed { problem: Problem::classify(&detail), detail }
119    }
120}
121
122/// A local inference server the engine found listening (see [`PRESETS`]).
123#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
124pub struct Detected {
125    /// The [`Preset::id`] whose port answered.
126    pub preset: String,
127    pub base_url: String,
128    /// Model ids it lists.
129    #[serde(default)]
130    pub models: Vec<String>,
131}
132
133/// Answer to `ClientMsg::TestInference` (only to the client that asked).
134#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
135pub struct InferenceTest {
136    /// What was tested, so a stale answer can be told apart.
137    pub base_url: String,
138    pub model: String,
139    /// The endpoint answered and has the model.
140    pub ok: bool,
141    #[serde(default)]
142    pub problem: Option<Problem>,
143    /// The raw error, or a short summary of what answered ("llama.cpp · 262k ctx").
144    #[serde(default)]
145    pub detail: String,
146    /// Model ids the server lists.
147    #[serde(default)]
148    pub models: Vec<String>,
149}
150
151/// A common setup to fill the connection form from.
152#[derive(Debug, Clone, Copy, PartialEq, Eq)]
153pub struct Preset {
154    pub id: &'static str,
155    pub name: &'static str,
156    pub base_url: &'static str,
157    /// A model name to start with when the server lists none.
158    pub model: &'static str,
159    /// Runs on your machine (so the engine looks for it at startup).
160    pub local: bool,
161    /// Needs an API key.
162    pub needs_key: bool,
163    pub hint: &'static str,
164}
165
166/// Common servers, local ones first. All speak OpenAI chat completions except Anthropic,
167/// which byob talks to over its Messages API (chosen from the URL).
168pub const PRESETS: &[Preset] = &[
169    Preset {
170        id: "llama-server",
171        name: "llama-server",
172        base_url: "http://127.0.0.1:8080/v1",
173        model: "default",
174        local: true,
175        needs_key: false,
176        hint: "llama.cpp's server: llama-server -m model.gguf --jinja",
177    },
178    Preset {
179        id: "ollama",
180        name: "Ollama",
181        base_url: "http://127.0.0.1:11434/v1",
182        model: "",
183        local: true,
184        needs_key: false,
185        hint: "ollama serve, then ollama pull a model",
186    },
187    Preset {
188        id: "lm-studio",
189        name: "LM Studio",
190        base_url: "http://127.0.0.1:1234/v1",
191        model: "",
192        local: true,
193        needs_key: false,
194        hint: "Start the server in LM Studio's Developer tab",
195    },
196    Preset {
197        id: "vllm",
198        name: "vLLM",
199        base_url: "http://127.0.0.1:8000/v1",
200        model: "",
201        local: true,
202        needs_key: false,
203        hint: "vllm serve MODEL --enable-auto-tool-choice",
204    },
205    Preset {
206        id: "openai",
207        name: "OpenAI",
208        base_url: "https://api.openai.com/v1",
209        model: "gpt-4.1-mini",
210        local: false,
211        needs_key: true,
212        hint: "Hosted: needs an API key",
213    },
214    Preset {
215        id: "anthropic",
216        name: "Anthropic",
217        base_url: "https://api.anthropic.com/v1",
218        model: "claude-opus-5-5",
219        local: false,
220        needs_key: true,
221        hint: "Claude, over Anthropic's Messages API: needs an API key",
222    },
223    Preset {
224        id: "openrouter",
225        name: "OpenRouter",
226        base_url: "https://openrouter.ai/api/v1",
227        model: "",
228        local: false,
229        needs_key: true,
230        hint: "Hosted, many models: needs an API key",
231    },
232];
233
234impl Preset {
235    pub fn by_id(id: &str) -> Option<&'static Preset> {
236        PRESETS.iter().find(|p| p.id == id)
237    }
238
239    /// The preset whose URL `base_url` is (ignoring a trailing slash and `localhost` vs `127.0.0.1`).
240    pub fn matching(base_url: &str) -> Option<&'static Preset> {
241        let norm = |u: &str| u.trim().trim_end_matches('/').replace("://localhost:", "://127.0.0.1:").to_ascii_lowercase();
242        let u = norm(base_url);
243        PRESETS.iter().find(|p| norm(p.base_url) == u)
244    }
245}
246
247#[cfg(test)]
248mod tests {
249    use super::*;
250
251    #[test]
252    fn classifies_llm_errors() {
253        let c = Problem::classify;
254        assert_eq!(c("could not reach http://127.0.0.1:9/v1/chat/completions: io: Connection refused"), Some(Problem::Unreachable));
255        assert_eq!(c("invalid inference URL: relative URL without a base"), Some(Problem::BadUrl));
256        assert_eq!(c("inference URL is invalid: x"), Some(Problem::BadUrl));
257        assert_eq!(c("server returned HTTP 401: Incorrect API key provided"), Some(Problem::Unauthorized));
258        assert_eq!(c("server returned HTTP 403: forbidden"), Some(Problem::Unauthorized));
259        assert_eq!(c("server returned HTTP 404: model \"qwen3\" not found, try pulling it first"), Some(Problem::ModelNotFound));
260        assert_eq!(c("server returned HTTP 400: The model `gpt-9` does not exist"), Some(Problem::ModelNotFound));
261        assert_eq!(c("server returned HTTP 404: File Not Found"), Some(Problem::WrongPath));
262        assert_eq!(c("server returned HTTP 429: slow down"), Some(Problem::RateLimited));
263        assert_eq!(c("server returned HTTP 503: Loading model"), Some(Problem::ServerError));
264        // Tool and context errors are not connection problems.
265        assert_eq!(c("server returned HTTP 400: context length exceeded"), None);
266        assert_eq!(c("stopped"), None);
267        assert_eq!(c("server returned HTTP abc"), None);
268        assert_eq!(c(""), None);
269    }
270
271    #[test]
272    fn every_problem_has_words() {
273        for p in [
274            Problem::BadUrl,
275            Problem::Unreachable,
276            Problem::Unauthorized,
277            Problem::WrongPath,
278            Problem::ModelNotFound,
279            Problem::RateLimited,
280            Problem::ServerError,
281        ] {
282            assert!(!p.title().is_empty() && p.advice().ends_with('.'));
283        }
284    }
285
286    #[test]
287    fn link_round_trips_and_failed_classifies() {
288        let l = Link::failed("could not reach http://x/v1/models: timeout");
289        assert_eq!(l, Link::Failed { problem: Some(Problem::Unreachable), detail: "could not reach http://x/v1/models: timeout".into() });
290        for l in [Link::Unknown, Link::Checking, Link::Ok, l] {
291            assert_eq!(serde_json::from_str::<Link>(&serde_json::to_string(&l).unwrap()).unwrap(), l);
292        }
293        assert_eq!(serde_json::to_string(&Link::Ok).unwrap(), r#"{"state":"ok"}"#);
294    }
295
296    #[test]
297    fn presets_match_urls() {
298        assert_eq!(Preset::matching("http://localhost:11434/v1/").map(|p| p.id), Some("ollama"));
299        assert_eq!(Preset::matching("http://127.0.0.1:8080/v1").map(|p| p.id), Some("llama-server"));
300        assert_eq!(Preset::matching("http://silicon:9931/v1"), None);
301        assert!(PRESETS.iter().all(|p| p.base_url.ends_with("/v1") && (p.local != p.needs_key)));
302        assert_eq!(Preset::by_id("openai").map(|p| p.needs_key), Some(true));
303        let claude = Preset::by_id("anthropic").map(|p| (p.base_url, p.model, p.needs_key));
304        assert_eq!(claude, Some(("https://api.anthropic.com/v1", "claude-opus-5-5", true)));
305    }
306}