nachalnik_providers/lib.rs
1//! Providers for the [`nachalnik`] agent runtime: somebody else's HTTP API, read carefully.
2//!
3//! The runtime ships no provider and never will: it has no network, and a kernel that opened a
4//! socket would have an opinion about who you talk to. Without this crate every adopter would write
5//! the same streamed HTTP client before asking a model anything.
6//!
7//! Everything in here is an [`Endpoint`]: an address, a key, a model identifier, a listing, and
8//! something to say when the server is slow. What splits the crate in two is whether the model
9//! behind it answers in *turns*.
10//!
11//! A [`Dialect`] does. It implements [`nachalnik::Provider`], so a kernel can be handed one and a
12//! session is what comes out. There are three, each behind the feature of the same name, and
13//! `openai` is on by default:
14//!
15//! - [`openai`] - `POST /chat/completions`, `choices[].delta`, tool calls assembled from fragments.
16//! What OpenRouter, ollama, vLLM, LM Studio, Together and most of the rest speak. With
17//! [`OpenAiCompatible::responses`], OpenAI's Responses API instead: ordered items, and a
18//! reasoning model's thinking sealed and sent back rather than redone every turn.
19//! - [`gemini`] - Google's own `generateContent`: `candidates[].content.parts`, whole calls, and
20//! ordered `thought` parts, which is the dialect [`nachalnik::Content::Blocks`] was built for.
21//! - [`anthropic`] - Anthropic's Messages API: typed blocks, whole calls whose arguments stream in
22//! fragments, and signed thinking that has to go back as it came. OpenRouter speaks it too.
23//!
24//! An `Endpoint` that is not a `Dialect` answers something other than a turn, and there is no
25//! kernel in its path at all:
26//!
27//! - [`system1`] - System One models, behind the `system1` feature: any of the ones OpenRouter
28//! serves, or a self-hosted engine answering the same route. Typed questions put to a state and
29//! answered with probabilities: no text, no tool calls, nothing to stream. It is for the
30//! decisions a program makes *around* a conversation rather than the conversation itself.
31//!
32//! ```no_run
33//! # use std::sync::Arc;
34//! # use nachalnik::{Config, Kernel};
35//! # #[cfg(feature = "openai")]
36//! # use nachalnik_providers::OpenAiCompatible;
37//! # #[cfg(feature = "openai")]
38//! # async fn go() -> Result<(), nachalnik::BoxError> {
39//! let provider = Arc::new(OpenAiCompatible::new(
40//! "openai/gpt-5",
41//! "https://openrouter.ai/api/v1",
42//! std::env::var("OPENROUTER_API_KEY")?,
43//! ));
44//! provider.probe().await;
45//!
46//! let kernel = Kernel::new(Config::default());
47//! kernel.set_provider(provider);
48//! # Ok(())
49//! # }
50//! ```
51//!
52//! note: nothing in this crate reads the environment. Where requests go, which key pays and what
53//! limit to measure against are the caller's to decide and say: a library that quietly picked up
54//! `OPENAI_API_KEY` would spend somebody's money on a variable exported for another reason.
55//! [`OpenAiCompatible::new`] takes the address and the key, and
56//! [`OpenAiCompatible::with_context_limit`] a limit to use instead of the advertised one.
57//!
58//! note: every dialect reports fragments through a [`nachalnik::DeltaSink`] and prints nothing: the
59//! screen belongs to whoever owns it.
60
61#![deny(unsafe_code)]
62#![deny(missing_docs)]
63
64use std::ops::RangeInclusive;
65
66#[cfg(any(feature = "anthropic", feature = "gemini", feature = "openai"))]
67use nachalnik::BoxError;
68use nachalnik::{Overrun, TooLong};
69
70mod endpoint;
71
72/// How many times a request is sent when the server says it is busy, including the first attempt.
73///
74/// note: counted per request, not on the provider. A counter every request draws from and any of
75/// them resets gives concurrent requests against a busy endpoint one allowance between them, so a
76/// request can fail its first attempt and give up without being retried at all.
77///
78/// note: here rather than in `waiting`, because the System One client asks as often as a dialect
79/// does and is built without either of them.
80#[cfg(any(
81 feature = "anthropic",
82 feature = "gemini",
83 feature = "openai",
84 feature = "system1"
85))]
86pub(crate) const RETRIES: usize = 4;
87
88/// How long a question about an endpoint is given: a listing of its models, or a probe for a
89/// model's context limit.
90///
91/// note: its own bound, because none of these is a turn. A turn is watched for silence and stopped
92/// by an interrupt. A startup, a `/model` or a `/models` awaits these with nothing watching, on a
93/// client that may have no timeout. An endpoint that takes the connection and never answers would
94/// hold whatever is waiting on it for ever. A timed-out question answers nothing, or no limit.
95#[cfg(any(feature = "anthropic", feature = "openai", feature = "gemini"))]
96pub(crate) const ASKING: std::time::Duration = std::time::Duration::from_secs(15);
97#[cfg(any(
98 feature = "anthropic",
99 feature = "gemini",
100 feature = "openai",
101 feature = "system1"
102))]
103mod markup;
104
105#[cfg(feature = "anthropic")]
106pub mod anthropic;
107#[cfg(any(feature = "anthropic", feature = "openai", feature = "system1"))]
108mod attribution;
109#[cfg(feature = "conformance")]
110pub mod conformance;
111
112#[cfg(feature = "gemini")]
113pub mod gemini;
114#[cfg(feature = "openai")]
115pub mod openai;
116#[cfg(any(feature = "anthropic", feature = "gemini", feature = "openai"))]
117pub(crate) mod reading;
118#[cfg(feature = "system1")]
119pub mod system1;
120#[cfg(any(feature = "anthropic", feature = "gemini", feature = "openai"))]
121pub(crate) mod waiting;
122
123#[cfg(feature = "anthropic")]
124pub use crate::anthropic::Anthropic;
125#[cfg(any(feature = "anthropic", feature = "openai", feature = "system1"))]
126pub use crate::attribution::Attribution;
127pub use crate::endpoint::{Dialect, Endpoint, Published};
128#[cfg(feature = "gemini")]
129pub use crate::gemini::Gemini;
130#[cfg(feature = "openai")]
131pub use crate::openai::OpenAiCompatible;
132#[cfg(feature = "system1")]
133pub use crate::system1::SystemOne;
134
135/// A transport failure, with every cause under it.
136///
137/// note: the error's own line shows only its category and the URL - `error sending request for url
138/// (…)`, or `builder error` - and the part a person can act on, `Connection refused` or `relative
139/// URL without a base`, is further down the chain. The kernel records an error as its `Display`, so
140/// a cause left in the chain is not shown. A layer that already repeats the one under it is not
141/// repeated again.
142#[cfg(any(
143 feature = "anthropic",
144 feature = "gemini",
145 feature = "openai",
146 feature = "system1"
147))]
148pub(crate) fn with_causes(e: &(dyn std::error::Error + 'static)) -> String {
149 let mut said = e.to_string();
150 let mut cause = e.source();
151 while let Some(under) = cause {
152 let words = under.to_string();
153 if !said.contains(&words) {
154 said.push_str(": ");
155 said.push_str(&words);
156 }
157 cause = under.source();
158 }
159
160 said
161}
162
163/// Whether an error means an account is out of free requests for the day, rather than having hit a
164/// momentary upstream limit.
165///
166/// note: both are the same HTTP status, and the difference matters to whoever is deciding what to
167/// do next. The first is worth skipping over - it will still be true in a minute - and the second
168/// is worth waiting out. A provider here already refuses to sit through a `Retry-After` longer than
169/// a minute for the same reason; this is for a caller reading the error afterwards and deciding
170/// whether the rest of a run is worth attempting.
171pub fn out_of_quota(error: &str) -> bool {
172 error.contains("per-day") || error.contains("daily")
173}
174
175/// Reads a server's complaint that a request was longer than the model takes, against what the
176/// caller already knows the model takes.
177///
178/// note: the wordings differ per vendor and the arithmetic does not, so this reads the numbers
179/// rather than the sentence, and reads exactly one of them: the request, the largest token count a
180/// complaint of this kind can name. The limit is not taken from the prose. In
181/// `you requested about 92674 tokens (92174 of text input, 500 in the output)`, against a model
182/// that takes 65536, the second-largest number is neither of the two, and the difference is what
183/// somebody would be told to free.
184///
185/// note: `limit` is what makes the reading safe: a refusal for length names a request larger than
186/// the model takes, and nothing else in the sentence is larger than the limit. Without one, a
187/// message naming a single number is left alone, because that number could be the request or the
188/// limit, and reading it the wrong way round calibrates a counter down on its way to a refusal.
189///
190/// note: the sentence has to name the limit, which is the one thing that says it counts in the same
191/// units as the session reading it. An aggregator normalises token counts into its own tokenizer
192/// and enforces its own window, but the model behind it refuses in its native one, and both
193/// refusals arrive from the same address for the same model id. The aggregator's refusal names the
194/// limit the session measures against and a request close to the session's own count, which is a
195/// correction worth having. The upstream's refusal names neither, because it is another tokenizer's
196/// count of the same bytes. Reading it would tell somebody to free tokens that were never there,
197/// and teach the counter a scale this session is not held to. An unread refusal is still the
198/// server's own sentence, which names the problem in words.
199pub fn too_long(said: &str, limit: Option<u64>) -> Option<TooLong> {
200 /// What names this kind of refusal, whoever phrased it.
201 const COMPLAINTS: [&str; 5] = [
202 "context length",
203 "context_length",
204 "too long",
205 "token count",
206 "tokens allowed",
207 ];
208 /// What a token count can be, either side of it. The upper bound is what keeps an identifier
209 /// out of the reading.
210 const PLAUSIBLE: RangeInclusive<u64> = 256..=100_000_000;
211 /// Beyond which a number is something other than a request that overshot.
212 const OVERSHOT_BY: u64 = 1_000;
213
214 let lowered = said.to_lowercase();
215 if !COMPLAINTS.iter().any(|which| lowered.contains(which)) {
216 return None;
217 }
218
219 let mut numbers: Vec<u64> = said
220 .split(|c: char| !c.is_ascii_digit())
221 .filter_map(|run| run.parse().ok())
222 .filter(|number| PLAUSIBLE.contains(number))
223 .collect();
224
225 numbers.sort_unstable();
226 numbers.dedup();
227
228 let tokens = numbers.pop()?;
229 let read = match limit {
230 Some(limit) => {
231 numbers.contains(&limit) && tokens > limit && tokens < limit.saturating_mul(OVERSHOT_BY)
232 }
233 // nothing to check it against, so the sentence has to name a second figure for the
234 // largest to be the request rather than the only number in it
235 None => !numbers.is_empty(),
236 };
237
238 read.then(|| TooLong {
239 overrun: Overrun { tokens, limit },
240 said: said.to_owned(),
241 })
242}
243
244/// The error a refused request becomes, reading a complaint about its length as one.
245///
246/// note: every dialect here ends at the same line: a sentence from a server, boxed and returned.
247/// This is that line, so the one refusal a session can act on arrives as [`TooLong`] rather than as
248/// prose. `TooLong` prints what the server said and nothing else.
249#[cfg(any(feature = "anthropic", feature = "gemini", feature = "openai"))]
250pub(crate) fn refused(said: String, limit: Option<usize>) -> BoxError {
251 match too_long(&said, limit.map(|limit| limit as u64)) {
252 Some(too_long) => too_long.into(),
253 None => said.into(),
254 }
255}
256
257/// A count of tokens an endpoint published, as a `usize`.
258///
259/// note: saturated rather than cast. On a 32-bit target a figure past `u32::MAX` wraps to a small
260/// one, and a listing's window read that way refuses every request as too long.
261#[cfg(any(feature = "anthropic", feature = "gemini", feature = "openai"))]
262pub(crate) fn tokens(count: u64) -> usize {
263 usize::try_from(count).unwrap_or(usize::MAX)
264}
265
266/// A call, from the text the model wrote for its arguments.
267///
268/// note: nothing written means no arguments, which is how a server spells a call to a tool that
269/// takes none. Read as JSON it failed, so the model was told its arguments were invalid and sent
270/// the same correct call again. Text that is not a JSON object is kept as written on
271/// [`ToolCall::unparsed`](nachalnik::ToolCall::unparsed): the kernel answers the call saying so,
272/// rather than a tool running on nothing, and the call goes back to the model empty rather than
273/// wrapped in a shape of ours that it would copy.
274#[cfg(any(feature = "anthropic", feature = "openai"))]
275pub(crate) fn called(
276 id: impl Into<nachalnik::ToolCallId>,
277 name: impl Into<String>,
278 written: &str,
279) -> nachalnik::ToolCall {
280 match written.trim() {
281 "" => nachalnik::ToolCall::new(id, name, serde_json::json!({})),
282 trimmed => match serde_json::from_str::<serde_json::Value>(trimmed) {
283 Ok(args @ serde_json::Value::Object(_)) => nachalnik::ToolCall::new(id, name, args),
284 _ => nachalnik::ToolCall::unparsed(id, name, written),
285 },
286 }
287}
288
289/// Whether a listed identifier names the model being asked about, allowing for the decorations a
290/// listing adds: Google's `models/` prefix and ollama's implicit `:latest` tag.
291pub fn same_model(listed: &str, model: &str) -> bool {
292 listed == model
293 || listed.strip_prefix("models/") == Some(model)
294 || listed.strip_suffix(":latest") == Some(model)
295}
296
297/// Three names from a listing, and an ellipsis where there are more.
298///
299/// note: three, because an endpoint that lists hundreds (OpenRouter does) would put every name on
300/// one line, leaving someone who mistyped a name nothing to correct it by.
301#[cfg(any(
302 feature = "anthropic",
303 feature = "gemini",
304 feature = "openai",
305 feature = "system1"
306))]
307pub(crate) fn some_of(listed: &[String]) -> String {
308 let some: Vec<&str> = listed.iter().take(3).map(String::as_str).collect();
309 match listed.len() > some.len() {
310 true => format!("{}, …", some.join(", ")),
311 false => some.join(", "),
312 }
313}
314
315/// The message when the model is not listed, and nothing when it is or the list is empty, which is
316/// no evidence either way.
317///
318/// note: nothing for an empty model either. An empty name is a session that has not picked a model,
319/// which the program already says in its own words. Treating it as unlisted says ` ` is not served
320/// on every `/restart` and `/endpoint` such a session made.
321#[cfg(any(feature = "anthropic", feature = "gemini", feature = "openai"))]
322pub(crate) fn unlisted(model: &str, listed: &[String]) -> Option<String> {
323 if !missing(model, listed) {
324 return None;
325 }
326
327 Some(format!(
328 "{model} is not one of the {} models this address lists ({})",
329 listed.len(),
330 some_of(listed)
331 ))
332}
333
334/// Whether `listed` is evidence that `model` is not served: a model is named, the list says
335/// something, and none of it is the model. The rule [`unlisted`] says in its words and System One's
336/// client in its own, which names the host.
337#[cfg(any(
338 feature = "anthropic",
339 feature = "gemini",
340 feature = "openai",
341 feature = "system1"
342))]
343pub(crate) fn missing(model: &str, listed: &[String]) -> bool {
344 !model.is_empty() && !listed.is_empty() && !listed.iter().any(|name| same_model(name, model))
345}
346
347/// Whether an address is OpenRouter's. Takes a whole URL or a bare authority.
348///
349/// note: one rule, in one place, because more than one thing in this workspace needs it, and each
350/// is a decision about a user's credentials or data: whether to send the app headers that put a
351/// program in a public ranking, and, in `kamchatka`, whether a session's own key may be spent on
352/// advice. Two copies of a host test can drift apart and still be read as the same rule.
353///
354/// note: on the authority alone, so a path, a port and a regional subdomain all still count, and
355/// `openrouter.ai.example.com` does not. A `contains` check would match that last one, which is why
356/// this is not one.
357///
358/// note: and on the host after any `user:password@`, which a request is sent with and not to.
359/// `https://openrouter.ai:x@example.com` goes to `example.com`; read up to its first `:`, it was
360/// OpenRouter's, so a key kept for OpenRouter went there too.
361pub fn is_openrouter(address: &str) -> bool {
362 let host = host_of(
363 address
364 .split_once("://")
365 .map_or(address, |(_, rest)| rest)
366 .split('/')
367 .next()
368 .unwrap_or_default(),
369 );
370
371 // a host name is not case-sensitive, and an address typed as `OpenRouter.AI` is OpenRouter's
372 let host = host.to_ascii_lowercase();
373
374 host == "openrouter.ai" || host.ends_with(".openrouter.ai")
375}
376
377/// The host an authority names: without the `user:password@` in front of it or the port after it.
378pub(crate) fn host_of(authority: &str) -> &str {
379 let host = authority
380 .rsplit_once('@')
381 .map_or(authority, |(_, host)| host);
382
383 host.split(':').next().unwrap_or(host)
384}
385
386/// A base URL as the requests are built on it: without the trailing `/` a copied address so often
387/// carries.
388///
389/// note: every path here is appended as `{base}/models` or `{base}/chat/completions`, so a trailing
390/// slash doubles, and a server that routes on the path answers the doubled one with a bare 404.
391/// Trimmed where an address comes in rather than where one is used, so that
392/// [`Endpoint::endpoint`] reports the address the requests actually go to.
393///
394/// note: never into the `//` after a scheme. `https://` is no address either way, but cut to
395/// `https:` it is one nobody typed.
396#[cfg(any(
397 feature = "anthropic",
398 feature = "gemini",
399 feature = "openai",
400 feature = "system1"
401))]
402pub(crate) fn address(url: impl Into<String>) -> String {
403 let mut url = url.into();
404 let kept = url.trim_end_matches('/').len();
405 if !url[..kept].ends_with(':') {
406 url.truncate(kept);
407 }
408 url
409}
410
411/// A request carrying a key, or none when the key is empty.
412///
413/// note: an empty key is how a caller says it has no key. ollama, llama.cpp and vLLM check nothing
414/// unless started with their own key. Sent anyway, an empty key is `Authorization: Bearer ` with
415/// nothing after it; a server that checks nothing ignores it, and a proxy in front may refuse.
416#[cfg(any(
417 feature = "anthropic",
418 feature = "gemini",
419 feature = "openai",
420 feature = "system1"
421))]
422pub(crate) trait Keyed {
423 /// The key as a bearer token, which is how the OpenAI dialect and System One take one.
424 #[cfg(any(feature = "openai", feature = "system1"))]
425 fn bearer(self, key: &str) -> Self;
426 /// The key in `x-api-key`, which is how Anthropic's API takes one.
427 #[cfg(feature = "anthropic")]
428 fn anthropic_key(self, key: &str) -> Self;
429 /// The key in `x-goog-api-key`, which is how Google's native API takes one.
430 #[cfg(any(feature = "gemini", feature = "openai"))]
431 fn google_key(self, key: &str) -> Self;
432}
433
434#[cfg(any(
435 feature = "anthropic",
436 feature = "gemini",
437 feature = "openai",
438 feature = "system1"
439))]
440impl Keyed for reqwest::RequestBuilder {
441 #[cfg(any(feature = "openai", feature = "system1"))]
442 fn bearer(self, key: &str) -> Self {
443 match key.is_empty() {
444 true => self,
445 false => self.bearer_auth(key),
446 }
447 }
448
449 #[cfg(feature = "anthropic")]
450 fn anthropic_key(self, key: &str) -> Self {
451 match key.is_empty() {
452 true => self,
453 false => self.header("x-api-key", key),
454 }
455 }
456
457 #[cfg(any(feature = "gemini", feature = "openai"))]
458 fn google_key(self, key: &str) -> Self {
459 match key.is_empty() {
460 true => self,
461 false => self.header("x-goog-api-key", key),
462 }
463 }
464}
465
466/// A base URL as [`ModelInfo::endpoint`](nachalnik::ModelInfo) records it, which goes into every
467/// record of a session.
468///
469/// note: credentials are removed. A `user:password@` before the host, or a key in the query string,
470/// is how some endpoints take one. A record is kept, shared and pasted into bug reports, where an
471/// address belongs and a key does not. The scheme, host and path say where a session was talking.
472#[cfg(any(feature = "anthropic", feature = "gemini", feature = "openai"))]
473pub(crate) fn recorded(url: &str) -> String {
474 let url = url.split(['?', '#']).next().unwrap_or_default();
475 let (scheme, rest) = match url.split_once("://") {
476 Some((scheme, rest)) => (Some(scheme), rest),
477 None => (None, url),
478 };
479 let (authority, path) = rest.split_at(rest.find('/').unwrap_or(rest.len()));
480 let host = authority
481 .rsplit_once('@')
482 .map_or(authority, |(_, host)| host);
483
484 match scheme {
485 Some(scheme) => format!("{scheme}://{host}{path}"),
486 None => format!("{host}{path}"),
487 }
488}
489
490/// Installs the `rustls` cryptographic provider, and does nothing if already installed.
491///
492/// note: reqwest is built with `rustls-no-provider`, so there is no default provider, and a client
493/// built without one fails at the first `https://` with "no process-level CryptoProvider
494/// available". Every constructor here calls this, so a test or example that never reaches `main`
495/// does not find out. It is public for a caller building its own [`reqwest::Client`], and for a
496/// program that installs a different provider before any of this runs.
497pub fn install_crypto() {
498 let _ = rustls::crypto::ring::default_provider().install_default();
499}
500
501#[cfg(test)]
502mod tests {
503 use super::*;
504
505 /// An address goes into the record without the credentials some endpoints take in it.
506 #[cfg(any(feature = "anthropic", feature = "gemini", feature = "openai"))]
507 #[test]
508 fn an_address_is_written_down_without_its_credentials() {
509 for (given, written) in [
510 (
511 "https://openrouter.ai/api/v1",
512 "https://openrouter.ai/api/v1",
513 ),
514 ("http://localhost:11434/v1", "http://localhost:11434/v1"),
515 (
516 "https://user:secret@proxy.example/v1",
517 "https://proxy.example/v1",
518 ),
519 (
520 "https://host.example/v1?key=secret#frag",
521 "https://host.example/v1",
522 ),
523 ("https://a@b@host.example", "https://host.example"),
524 ("localhost:8000/v1", "localhost:8000/v1"),
525 ] {
526 assert_eq!(recorded(given), written, "{given}");
527 }
528 }
529
530 /// A model the address does not list is said with up to three of the names it does, and an
531 /// ellipsis only where there are more.
532 #[cfg(any(feature = "anthropic", feature = "gemini", feature = "openai"))]
533 #[test]
534 fn a_model_not_listed_is_said_with_some_of_the_listing() {
535 let listed = |names: &[&str]| names.iter().map(|it| (*it).to_owned()).collect::<Vec<_>>();
536
537 assert_eq!(
538 unlisted("m", &listed(&["a", "b", "c"])).as_deref(),
539 Some("m is not one of the 3 models this address lists (a, b, c)")
540 );
541 assert_eq!(
542 unlisted("m", &listed(&["a", "b", "c", "d"])).as_deref(),
543 Some("m is not one of the 4 models this address lists (a, b, c, …)")
544 );
545 }
546
547 /// A listed name is the model under either decoration a listing puts on it.
548 ///
549 /// note: each decoration is the whole difference for one kind of endpoint. Google lists every
550 /// model under `models/`, and ollama tags a bare name with `:latest`. A match that needed both
551 /// would tell every such session its model is not served.
552 #[test]
553 fn a_listed_name_is_the_model_under_either_decoration() {
554 for listed in ["resident", "models/resident", "resident:latest"] {
555 assert!(same_model(listed, "resident"), "{listed}");
556 }
557 for listed in ["models/other", "other:latest"] {
558 assert!(!same_model(listed, "resident"), "{listed}");
559 }
560 }
561
562 /// Whose address it is, from a whole URL or a bare authority.
563 ///
564 /// note: it decides app attribution, and whether `kamchatka` may spend a session's key on
565 /// advice. The second is why both URL forms are tested. `Attribution::sign` passes a bare host,
566 /// and `kamchatka` passes the whole address.
567 ///
568 /// note: `openrouter.ai.example.com` decides the shape of the function. A `contains` or suffix
569 /// test on the raw address would match it, and callers would name a program and spend a key
570 /// against a host that merely put somebody else's name in front of its own.
571 #[test]
572 fn an_address_is_openrouters_by_its_authority_and_nothing_else() {
573 for theirs in [
574 "openrouter.ai",
575 "openrouter.ai:443",
576 "api.openrouter.ai",
577 "https://openrouter.ai/api/v1",
578 "https://openrouter.ai/api/alpha",
579 "https://openrouter.ai",
580 "https://user:secret@openrouter.ai/api/v1",
581 "https://OpenRouter.AI/api/v1",
582 "API.OPENROUTER.AI",
583 ] {
584 assert!(is_openrouter(theirs), "{theirs}");
585 }
586
587 for not in [
588 "localhost:11434",
589 "http://localhost:11434/v1",
590 "https://generativelanguage.googleapis.com/v1beta",
591 "https://api.typesafe.ai/v1",
592 "openrouter.ai.example.com",
593 "https://openrouter.ai.example.com/api/v1",
594 "notopenrouter.ai",
595 "https://openrouter.ai:x@example.com/api/v1",
596 "https://openrouter.ai@example.com/api/v1",
597 "",
598 ] {
599 assert!(!is_openrouter(not), "{not}");
600 }
601 }
602
603 /// Vendors phrase the same complaint differently; the arithmetic is the same.
604 ///
605 /// note: each is a sentence an endpoint sent. The first carries a status code, which is a
606 /// number in the message and not a token count. The second carries three numbers, of which the
607 /// second-largest is neither the request nor the limit. What is read from the prose is the
608 /// request alone.
609 #[test]
610 fn a_refusal_for_length_is_read_whoever_phrased_it() {
611 let shapes = [
612 (
613 "400 Bad Request: Provider returned error - {\"error\":{\"message\":\"The \
614 request is 286315 tokens long and exceeds this model's context length of \
615 262144 tokens.\",\"code\":\"context_length_exceeded\"}}",
616 262_144,
617 286_315,
618 ),
619 (
620 "400 Bad Request: This endpoint's maximum context length is 65536 tokens. \
621 However, you requested about 92674 tokens (92174 of text input, 500 in the \
622 output). Please reduce the length of either one.",
623 65_536,
624 92_674,
625 ),
626 (
627 "This model's maximum context length is 8192 tokens. However, your messages \
628 resulted in 10000 tokens. Please reduce the length of the messages.",
629 8_192,
630 10_000,
631 ),
632 (
633 "prompt is too long: 250000 tokens > 200000 maximum",
634 200_000,
635 250_000,
636 ),
637 (
638 "The input token count (1050000) exceeds the maximum number of tokens allowed \
639 (1048576).",
640 1_048_576,
641 1_050_000,
642 ),
643 // the aggregator refusing in its own units, and the reading this is for: the counter
644 // had put the same request at 71231, which is the correction arriving
645 (
646 "400 Bad Request: This endpoint's maximum context length is 65536 tokens. \
647 However, you requested about 71311 tokens (70014 of text input, 1297 of tool \
648 input). Please reduce the length of either one, or use the context-compression \
649 plugin to compress your prompt automatically.",
650 65_536,
651 71_311,
652 ),
653 ];
654
655 for (said, limit, tokens) in shapes {
656 let read = too_long(said, Some(limit)).unwrap_or_else(|| panic!("not read: {said}"));
657 assert_eq!(
658 read.overrun,
659 Overrun {
660 tokens,
661 limit: Some(limit)
662 },
663 "read wrongly: {said}"
664 );
665 assert_eq!(read.to_string(), said, "the sentence survives the reading");
666
667 // and the same sentence from an endpoint that never said what the model takes: the
668 // request is still there to be read, and there is nothing to check it against
669 assert_eq!(
670 too_long(said, None).map(|read| read.overrun.tokens),
671 Some(tokens),
672 "not read without a limit: {said}"
673 );
674 }
675 }
676
677 /// What is not a refusal for length, and what is one but says too little to act on.
678 #[test]
679 fn a_refusal_that_cannot_be_read_is_left_as_a_sentence() {
680 let nothing = [
681 // not about length at all, and the second one has two numbers in it
682 ("429 Too Many Requests: rate limit exceeded", Some(8_192)),
683 (
684 "401 Unauthorized: key 12345678 is not valid for model 4096",
685 Some(8_192),
686 ),
687 // about length, naming only what the model takes. With a limit, it fails the check.
688 // Without one, a lone number could be either value, and reading it wrong is worse than
689 // not reading it
690 ("This model's maximum context length is 8192 tokens.", None),
691 (
692 "This model's maximum context length is 8192 tokens.",
693 Some(8_192),
694 ),
695 // about length, and the only number in it that could be a count is an identifier
696 (
697 "context length exceeded on request 987654321 (limit 8192)",
698 Some(8_192),
699 ),
700 // about length, and counted by somebody else: the same model id at the same address
701 // as the fixture above, refused by the model behind the aggregator rather than by the
702 // aggregator, in the model's own tokenizer. Every number here is true and none of
703 // them is in the units this session is held to
704 (
705 "400 Bad Request: Provider returned error - {\"error\":{\"message\":\"This \
706 model's maximum context length is 131072 tokens. However, you requested 8192 \
707 output tokens and your prompt contains at least 122881 input tokens, for a \
708 total of at least 131073 tokens.\"}}",
709 Some(65_536),
710 ),
711 ];
712
713 for (said, limit) in nothing {
714 assert_eq!(too_long(said, limit), None, "read anyway: {said}");
715 }
716 }
717}