#![cfg_attr(
not(all(feature = "openai", feature = "anthropic", feature = "gemini", feature = "bedrock")),
allow(dead_code)
)]
use std::collections::VecDeque;
use std::time::Duration;
use futures::StreamExt;
use crate::error::LlmError;
use crate::sse::{SseEvent, SseParser};
use crate::types::{ChatDelta, ChatStream};
pub(crate) const MAX_ERROR_BODY_BYTES: usize = 64 * 1024;
pub(crate) fn build_client(connect_timeout: Duration) -> Result<reqwest::Client, LlmError> {
let _already_installed = rustls::crypto::ring::default_provider().install_default();
reqwest::Client::builder()
.connect_timeout(connect_timeout)
.redirect(reqwest::redirect::Policy::none())
.build()
.map_err(|e| LlmError::Config(format!("cannot build HTTP client: {e}")))
}
use crate::credentials::is_secure_endpoint;
fn origin(url: &str) -> String {
let (scheme, rest) = url.split_once("://").unwrap_or(("", url));
let authority = rest.split(['/', '?', '#']).next().unwrap_or_default();
format!("{}://{}", scheme.to_ascii_lowercase(), authority.to_ascii_lowercase())
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct Endpoint {
pub base_url: String,
pub send_static: bool,
}
pub(crate) fn resolve_endpoint(configured: &str, credential_base: Option<&str>) -> Result<Endpoint, LlmError> {
match credential_base.filter(|u| !u.is_empty()) {
None => Ok(Endpoint { base_url: configured.to_owned(), send_static: true }),
Some(url) => {
if !is_secure_endpoint(url) {
return Err(LlmError::Auth(
"credential supplied a non-https base_url; refusing to send credentials there".into(),
));
}
let send_static = origin(url) == origin(configured);
Ok(Endpoint { base_url: url.to_owned(), send_static })
}
}
}
#[derive(Debug, Default)]
pub(crate) struct Decoded {
pub deltas: Vec<ChatDelta>,
pub done: bool,
}
impl Decoded {
pub fn deltas(deltas: Vec<ChatDelta>) -> Self {
Self { deltas, done: false }
}
pub fn done(deltas: Vec<ChatDelta>) -> Self {
Self { deltas, done: true }
}
}
pub(crate) trait SseDecoder: Send + 'static {
fn decode(&mut self, event: SseEvent) -> Result<Decoded, LlmError>;
fn end_of_stream(&mut self) -> Result<Vec<ChatDelta>, LlmError> {
Err(truncated())
}
}
pub(crate) fn truncated() -> LlmError {
LlmError::Network("stream ended unexpectedly before the response was complete".into())
}
fn retry_after_ms(headers: &reqwest::header::HeaderMap) -> Option<u64> {
let raw = headers.get("retry-after-ms").or_else(|| headers.get(reqwest::header::RETRY_AFTER))?;
let text = raw.to_str().ok()?.trim();
if headers.contains_key("retry-after-ms") {
return text.parse::<u64>().ok();
}
let seconds: f64 = text.parse().ok()?;
if seconds.is_finite() && seconds >= 0.0 {
#[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)]
Some((seconds * 1000.0) as u64)
} else {
None
}
}
fn network_error(e: reqwest::Error) -> LlmError {
LlmError::Network(e.without_url().to_string())
}
async fn read_error_body(response: reqwest::Response) -> String {
let mut bytes: Vec<u8> = Vec::new();
let mut stream = response.bytes_stream();
while let Some(chunk) = stream.next().await {
match chunk {
Ok(chunk) => {
let room = MAX_ERROR_BODY_BYTES.saturating_sub(bytes.len());
bytes.extend(chunk.iter().take(room));
if bytes.len() >= MAX_ERROR_BODY_BYTES {
break;
}
}
Err(e) => {
let partial = String::from_utf8_lossy(&bytes).into_owned();
return format!("{partial} (error body could not be read completely: {})", e.without_url());
}
}
}
String::from_utf8_lossy(&bytes).into_owned()
}
pub(crate) async fn send_sse<D: SseDecoder>(
request: reqwest::RequestBuilder,
decoder: D,
) -> Result<ChatStream, LlmError> {
let response = send_request(request, LlmError::from_status).await?;
Ok(sse_stream(response, decoder))
}
pub(crate) async fn send_request(
request: reqwest::RequestBuilder,
classify: impl FnOnce(u16, &str, Option<u64>) -> LlmError,
) -> Result<reqwest::Response, LlmError> {
let response = request.send().await.map_err(network_error)?;
let status = response.status();
if status.is_redirection() {
return Err(LlmError::BadRequest(format!(
"provider endpoint answered with a redirect (HTTP {}); redirects are not followed so credentials \
cannot leak to another host — point `base_url` at the final endpoint",
status.as_u16()
)));
}
if !status.is_success() {
let retry = retry_after_ms(response.headers());
let body = read_error_body(response).await;
return Err(classify(status.as_u16(), &body, retry));
}
Ok(response)
}
pub(crate) fn byte_stream(
response: reqwest::Response,
) -> futures::stream::BoxStream<'static, Result<Vec<u8>, LlmError>> {
response.bytes_stream().map(|r| r.map(|b| b.to_vec()).map_err(network_error)).boxed()
}
pub(crate) fn sse_stream<D: SseDecoder>(response: reqwest::Response, decoder: D) -> ChatStream {
struct State<D> {
bytes: futures::stream::BoxStream<'static, Result<Vec<u8>, LlmError>>,
parser: SseParser,
decoder: D,
pending: VecDeque<Result<ChatDelta, LlmError>>,
finished: bool,
}
impl<D: SseDecoder> State<D> {
fn feed(&mut self, events: Vec<SseEvent>) -> bool {
for event in events {
match self.decoder.decode(event) {
Ok(decoded) => {
self.pending.extend(decoded.deltas.into_iter().map(Ok));
if decoded.done {
return true;
}
}
Err(e) => {
self.pending.push_back(Err(e));
return true;
}
}
}
false
}
fn fail(&mut self, error: LlmError) {
self.pending.push_back(Err(error));
self.finished = true;
}
}
let bytes = byte_stream(response);
let state = State { bytes, parser: SseParser::new(), decoder, pending: VecDeque::new(), finished: false };
let stream = futures::stream::unfold(state, |mut st| async move {
loop {
if let Some(item) = st.pending.pop_front() {
return Some((item, st));
}
if st.finished {
return None;
}
match st.bytes.next().await {
Some(Ok(chunk)) => match st.parser.push(&chunk) {
Ok(events) => st.finished = st.feed(events),
Err(e) => st.fail(LlmError::Decode(e.to_string())),
},
Some(Err(e)) => st.fail(e),
None => {
let tail: Vec<SseEvent> = match st.parser.finish() {
Ok(tail) => tail.into_iter().collect(),
Err(e) => {
st.fail(LlmError::Decode(e.to_string()));
continue;
}
};
if st.feed(tail) {
st.finished = true;
continue;
}
match st.decoder.end_of_stream() {
Ok(deltas) => {
st.pending.extend(deltas.into_iter().map(Ok));
st.finished = true;
}
Err(e) => st.fail(e),
}
}
}
}
});
Box::pin(stream)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn retry_after_parsing() {
let mut h = reqwest::header::HeaderMap::new();
h.insert(reqwest::header::RETRY_AFTER, reqwest::header::HeaderValue::from_static("2"));
assert_eq!(retry_after_ms(&h), Some(2000));
h.insert("retry-after-ms", reqwest::header::HeaderValue::from_static("150"));
assert_eq!(retry_after_ms(&h), Some(150));
}
#[test]
fn secure_endpoints() {
assert!(is_secure_endpoint("https://api.example.com/v1"));
assert!(is_secure_endpoint("http://localhost:8080/v1"));
assert!(is_secure_endpoint("http://127.0.0.1:9/x"));
assert!(is_secure_endpoint("http://[::1]:9/x"));
assert!(is_secure_endpoint("http://10.0.2.2:8787"));
assert!(!is_secure_endpoint("http://api.example.com"));
assert!(!is_secure_endpoint("http://localhost.evil.com"));
assert!(!is_secure_endpoint("ftp://x"));
}
#[test]
fn endpoint_override_drops_static_headers_cross_origin() {
let e = resolve_endpoint("https://api.openai.com/v1", None).unwrap();
assert!(e.send_static);
let e = resolve_endpoint("https://api.openai.com/v1", Some("https://proxy.example.com/v1")).unwrap();
assert_eq!(e.base_url, "https://proxy.example.com/v1");
assert!(!e.send_static);
let e = resolve_endpoint("https://proxy.example.com/v1", Some("https://PROXY.example.com/v2")).unwrap();
assert!(e.send_static);
assert!(resolve_endpoint("https://api.openai.com/v1", Some("http://attacker.example")).is_err());
}
#[test]
fn truncation_is_retryable() {
assert!(truncated().is_retryable());
}
}