switchyard_llm_client/
backend.rs

1// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Per-provider backend configuration: wire format, upstream URL, and auth.
5
6use std::{collections::BTreeMap, fmt};
7
8use reqwest::RequestBuilder;
9use serde_json::Value;
10use switchyard_protocol::WireFormat;
11
12use crate::error::is_overflow_body;
13
14const ANTHROPIC_VERSION: &str = "2023-06-01";
15
16/// Default number of retries for server-configured upstream calls.
17pub const DEFAULT_MAX_RETRIES: u32 = 2;
18
19// Canonical OpenAI phrase plus NVIDIA/LiteLLM wrap variants. Adding a new
20// provider-wrap is a one-line entry here, not a fork of the parsing logic.
21const OPENAI_OVERFLOW_PHRASES: &[&str] = &[
22    "maximum context length",
23    "context length exceeded",
24    "context window",
25    "context length is only",
26    "please reduce the length of the input",
27    "exceeds the maximum allowed input length",
28];
29
30// Anthropic has no structured `error.code`, so detection is phrase-based only.
31const ANTHROPIC_OVERFLOW_PHRASES: &[&str] = &[
32    "prompt is too long",
33    "maximum number of tokens",
34    "context window",
35    "context length",
36];
37
38/// Shared HTTP configuration for one upstream backend.
39#[derive(Clone)]
40pub struct HttpBackendConfig {
41    /// Base URL of the provider API (e.g. `https://api.openai.com/v1`).
42    pub base_url: String,
43    /// API key for the provider, loaded by the caller. `None` sends no auth.
44    pub api_key: Option<String>,
45    /// Static headers added to every outbound call to this backend.
46    pub extra_headers: BTreeMap<String, String>,
47    /// Default top-level request fields, applied only when the request omits the key.
48    pub extra_body: BTreeMap<String, Value>,
49    /// Additional attempts after the initial upstream request.
50    pub max_retries: u32,
51}
52
53impl fmt::Debug for HttpBackendConfig {
54    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
55        f.debug_struct("HttpBackendConfig")
56            .field("base_url", &self.base_url)
57            .field("api_key", &self.api_key.as_ref().map(|_| "[REDACTED]"))
58            .field("extra_headers", &self.extra_headers)
59            .field("extra_body_keys", &self.extra_body.keys())
60            .field("max_retries", &self.max_retries)
61            .finish()
62    }
63}
64
65/// A configured upstream backend, one variant per built-in wire format.
66///
67/// The variant fixes the wire format, URL path, and auth scheme together so no
68/// invalid combination can be constructed.
69#[derive(Clone, Debug)]
70pub enum Backend {
71    /// OpenAI-compatible Chat Completions API.
72    OpenAiChat(HttpBackendConfig),
73    /// OpenAI Responses API.
74    OpenAiResponses(HttpBackendConfig),
75    /// Anthropic Messages API.
76    Anthropic(HttpBackendConfig),
77}
78
79impl Backend {
80    /// The wire format the request IR is encoded to for this backend.
81    pub fn wire_format(&self) -> WireFormat {
82        match self {
83            Backend::OpenAiChat(_) => WireFormat::OpenAiChat,
84            Backend::OpenAiResponses(_) => WireFormat::OpenAiResponses,
85            Backend::Anthropic(_) => WireFormat::AnthropicMessages,
86        }
87    }
88
89    // Shared HTTP config, regardless of variant.
90    fn config(&self) -> &HttpBackendConfig {
91        match self {
92            Backend::OpenAiChat(config)
93            | Backend::OpenAiResponses(config)
94            | Backend::Anthropic(config) => config,
95        }
96    }
97
98    /// The fully resolved upstream URL for this backend's endpoint.
99    ///
100    /// Tolerates base URLs that already include the provider path (or a bare
101    /// `/v1`), matching the join rules of the existing native backends.
102    pub fn url(&self) -> String {
103        let base_url = self.config().base_url.trim_end_matches('/');
104        match self {
105            Backend::OpenAiChat(_) => openai_url(base_url, "/chat/completions"),
106            Backend::OpenAiResponses(_) => openai_url(base_url, "/responses"),
107            Backend::Anthropic(_) => anthropic_url(base_url),
108        }
109    }
110
111    /// Applies this backend's auth and version headers to a request builder.
112    ///
113    /// OpenAI variants use `Authorization: Bearer <key>`; Anthropic uses
114    /// `x-api-key: <key>` plus the required `anthropic-version` header.
115    pub fn apply_auth(&self, mut builder: RequestBuilder) -> RequestBuilder {
116        let api_key = self.config().api_key.as_deref();
117        match self {
118            Backend::OpenAiChat(_) | Backend::OpenAiResponses(_) => {
119                if let Some(api_key) = api_key {
120                    builder = builder.bearer_auth(api_key);
121                }
122            }
123            Backend::Anthropic(_) => {
124                builder = builder.header("anthropic-version", ANTHROPIC_VERSION);
125                if let Some(api_key) = api_key {
126                    builder = builder.header("x-api-key", api_key);
127                }
128            }
129        }
130        builder
131    }
132
133    /// Static per-backend headers to forward on every call.
134    pub fn extra_headers(&self) -> &BTreeMap<String, String> {
135        &self.config().extra_headers
136    }
137
138    /// Default top-level fields to merge into outbound request bodies.
139    pub fn extra_body(&self) -> &BTreeMap<String, Value> {
140        &self.config().extra_body
141    }
142
143    /// Additional attempts allowed after the initial request.
144    pub fn max_retries(&self) -> u32 {
145        self.config().max_retries
146    }
147
148    /// Whether this backend speaks the Anthropic Messages wire format — the only
149    /// one with a `count_tokens` endpoint.
150    pub fn is_anthropic(&self) -> bool {
151        matches!(self, Backend::Anthropic(_))
152    }
153
154    /// The upstream `/v1/messages/count_tokens` URL, derived from the same base
155    /// URL join as [`url`](Self::url).
156    pub fn count_tokens_url(&self) -> String {
157        let base_url = self.config().base_url.trim_end_matches('/');
158        format!("{}/count_tokens", anthropic_url(base_url))
159    }
160
161    /// Whether an upstream 400 `body` looks like a context-window overflow for
162    /// this backend's provider.
163    pub(crate) fn is_context_overflow(&self, body: &str) -> bool {
164        match self {
165            Backend::OpenAiChat(_) | Backend::OpenAiResponses(_) => is_overflow_body(
166                body,
167                |value| {
168                    value
169                        .get("error")
170                        .and_then(|err| err.get("code"))
171                        .and_then(serde_json::Value::as_str)
172                        == Some("context_length_exceeded")
173                },
174                OPENAI_OVERFLOW_PHRASES,
175            ),
176            Backend::Anthropic(_) => is_overflow_body(body, |_| false, ANTHROPIC_OVERFLOW_PHRASES),
177        }
178    }
179}
180
181// Accept either a root `/v1` URL or an already-specific OpenAI endpoint URL.
182fn openai_url(base_url: &str, suffix: &str) -> String {
183    let base_root = base_url
184        .strip_suffix("/chat/completions")
185        .or_else(|| base_url.strip_suffix("/responses"))
186        .unwrap_or(base_url);
187    format!("{base_root}{suffix}")
188}
189
190// Accept a bare host, a `/v1` root, or an already-specific `/v1/messages` URL.
191fn anthropic_url(base_url: &str) -> String {
192    if base_url.ends_with("/v1/messages") {
193        base_url.to_string()
194    } else if base_url.ends_with("/v1") {
195        format!("{base_url}/messages")
196    } else {
197        format!("{base_url}/v1/messages")
198    }
199}
200
201#[cfg(test)]
202mod tests {
203    use super::*;
204
205    fn config(base_url: &str) -> HttpBackendConfig {
206        HttpBackendConfig {
207            base_url: base_url.to_string(),
208            api_key: Some("secret".to_string()),
209            extra_headers: BTreeMap::new(),
210            extra_body: BTreeMap::new(),
211            max_retries: 0,
212        }
213    }
214
215    #[test]
216    fn openai_chat_url_joins_bare_v1() {
217        let backend = Backend::OpenAiChat(config("https://api.openai.com/v1"));
218        assert_eq!(backend.url(), "https://api.openai.com/v1/chat/completions");
219    }
220
221    #[test]
222    fn openai_chat_url_tolerates_trailing_slash_and_existing_suffix() {
223        assert_eq!(
224            Backend::OpenAiChat(config("https://api.openai.com/v1/")).url(),
225            "https://api.openai.com/v1/chat/completions"
226        );
227        assert_eq!(
228            Backend::OpenAiChat(config("https://api.openai.com/v1/chat/completions")).url(),
229            "https://api.openai.com/v1/chat/completions"
230        );
231    }
232
233    #[test]
234    fn openai_responses_url_uses_responses_path() {
235        assert_eq!(
236            Backend::OpenAiResponses(config("https://api.openai.com/v1")).url(),
237            "https://api.openai.com/v1/responses"
238        );
239    }
240
241    #[test]
242    fn anthropic_url_join_cases() {
243        assert_eq!(
244            Backend::Anthropic(config("https://api.anthropic.com")).url(),
245            "https://api.anthropic.com/v1/messages"
246        );
247        assert_eq!(
248            Backend::Anthropic(config("https://api.anthropic.com/v1")).url(),
249            "https://api.anthropic.com/v1/messages"
250        );
251        assert_eq!(
252            Backend::Anthropic(config("https://api.anthropic.com/v1/messages")).url(),
253            "https://api.anthropic.com/v1/messages"
254        );
255    }
256
257    #[test]
258    fn count_tokens_url_joins_every_base_url_shape() {
259        assert_eq!(
260            Backend::Anthropic(config("https://host")).count_tokens_url(),
261            "https://host/v1/messages/count_tokens"
262        );
263        assert_eq!(
264            Backend::Anthropic(config("https://host/v1")).count_tokens_url(),
265            "https://host/v1/messages/count_tokens"
266        );
267        assert_eq!(
268            Backend::Anthropic(config("https://host/v1/messages")).count_tokens_url(),
269            "https://host/v1/messages/count_tokens"
270        );
271        // Trailing slash is trimmed before the join.
272        assert_eq!(
273            Backend::Anthropic(config("https://host/v1/")).count_tokens_url(),
274            "https://host/v1/messages/count_tokens"
275        );
276    }
277
278    #[test]
279    fn only_anthropic_backend_is_anthropic() {
280        assert!(Backend::Anthropic(config("x")).is_anthropic());
281        assert!(!Backend::OpenAiChat(config("x")).is_anthropic());
282        assert!(!Backend::OpenAiResponses(config("x")).is_anthropic());
283    }
284
285    #[test]
286    fn wire_format_matches_variant() {
287        assert_eq!(
288            Backend::OpenAiChat(config("x")).wire_format(),
289            WireFormat::OpenAiChat
290        );
291        assert_eq!(
292            Backend::OpenAiResponses(config("x")).wire_format(),
293            WireFormat::OpenAiResponses
294        );
295        assert_eq!(
296            Backend::Anthropic(config("x")).wire_format(),
297            WireFormat::AnthropicMessages
298        );
299    }
300
301    #[test]
302    fn openai_detects_canonical_and_wrapped_overflow() {
303        let backend = Backend::OpenAiChat(config("x"));
304        assert!(
305            backend.is_context_overflow(
306                r#"{"error":{"code":"context_length_exceeded","message":"x"}}"#
307            )
308        );
309        // NVIDIA/LiteLLM message wrap with no structured code.
310        assert!(backend.is_context_overflow(
311            r#"{"error":{"message":"the model's context length is only 131072 tokens"}}"#
312        ));
313        assert!(!backend.is_context_overflow(r#"{"error":{"code":"invalid_api_key"}}"#));
314        // Hub GLM (LiteLLM-wrapped): code is "400", detection relies on phrase match.
315        assert!(backend.is_context_overflow(
316            r#"{"error":{"message":"Input length 877338 exceeds the maximum allowed input length of 639968 tokens","code":"400"}}"#
317        ));
318    }
319
320    #[test]
321    fn anthropic_detects_prompt_too_long() {
322        let backend = Backend::Anthropic(config("x"));
323        assert!(
324            backend.is_context_overflow(
325                r#"{"error":{"message":"prompt is too long: 200000 tokens"}}"#
326            )
327        );
328        assert!(!backend.is_context_overflow(r#"{"error":{"message":"overloaded"}}"#));
329    }
330}
switchyard_llm_client/backend.rs

switchyard_llm_client/
backend.rs
