Skip to main content

jev_client/
http.rs

1//! The real transport: HTTPS through `reqwest` and `rustls`, with retries.
2
3use std::fmt;
4use std::sync::Arc;
5use std::time::Duration;
6
7use reqwest::header::{ACCEPT, AUTHORIZATION, CONTENT_TYPE, HeaderMap, HeaderValue, USER_AGENT};
8use reqwest::{Method, Url};
9use rustls_platform_verifier::BuilderVerifierExt;
10use serde::de::DeserializeOwned;
11use zeroize::Zeroizing;
12
13use crate::base_url::BaseUrl;
14use crate::clock::{Clock, SystemClock};
15use crate::error::{Error, ErrorKind};
16use crate::models::ModelList;
17use crate::request::Request;
18use crate::response::Response;
19use crate::retry::{RetryPolicy, server_delay};
20use crate::secret::ApiKey;
21use crate::throttle::Throttle;
22use crate::transport::{Reply, ReplyMeta, Transport};
23
24/// The response header carrying the id of a request.
25const REQUEST_ID_HEADER: &str = "x-typesafe-request-id";
26
27/// The largest response body that is read. A real response is a few kilobytes; the cap stops a
28/// misbehaving server or proxy from exhausting memory.
29const MAX_RESPONSE_BYTES: usize = 32 * 1024 * 1024;
30
31/// The time allowed to establish a connection, within the per-attempt timeout.
32const CONNECT_TIMEOUT: Duration = Duration::from_secs(10);
33
34/// Tracing target for request and response bodies, which are logged only on explicit opt-in.
35const BODY_LOG_TARGET: &str = "jev_client::http::body";
36
37/// A [`Transport`] that talks to the TypeSafe API over HTTPS.
38///
39/// - TLS is `rustls` with the operating system's certificate verifier. Verification cannot be
40///   switched off, and a plain `http://` base URL is only possible as described on [`BaseUrl`].
41/// - Redirects are not followed, so the API key is never forwarded to another host.
42/// - Failures are retried as the [`RetryPolicy`] says, which by default matches the official SDKs.
43/// - At `DEBUG` level it traces the method, URL, status, timing, request id and every retry
44///   decision. Bodies are traced only after [`HttpTransportBuilder::log_bodies`], because a
45///   request's `state` is often customer data. The `Authorization` header is never traced.
46///
47/// It must be used inside a Tokio runtime.
48///
49/// ```no_run
50/// # async fn example() -> Result<(), Box<dyn std::error::Error>> {
51/// use jev_client::{ApiKey, HttpTransport, Noul, Request, Transport};
52///
53/// let key = ApiKey::new(std::env::var("TYPESAFE_API_KEY")?)?;
54/// let transport = HttpTransport::builder(key).build()?;
55///
56/// let request = Request::new("Help! My payouts have been failing for 3 days.", "jev-latest")
57///     .question("is_urgent", Noul::new("Does this convey urgency?"));
58/// let reply = transport.evaluate(&request).await?;
59///
60/// println!("{} answered in {:?}", reply.body.model, reply.meta.latency);
61/// # Ok(())
62/// # }
63/// ```
64#[derive(Clone)]
65pub struct HttpTransport {
66    client: reqwest::Client,
67    base_url: BaseUrl,
68    api_key: ApiKey,
69    authorization: HeaderValue,
70    retry: RetryPolicy,
71    clock: Arc<dyn Clock>,
72    log_bodies: bool,
73    throttle: Option<Arc<Throttle>>,
74}
75
76impl HttpTransport {
77    /// Starts building a transport that authenticates with `api_key`.
78    #[must_use]
79    pub fn builder(api_key: ApiKey) -> HttpTransportBuilder {
80        HttpTransportBuilder {
81            api_key,
82            base_url: BaseUrl::default(),
83            retry: RetryPolicy::default(),
84            clock: Arc::new(SystemClock),
85            user_agent: None,
86            log_bodies: false,
87        }
88    }
89
90    /// The API root this transport talks to.
91    #[must_use]
92    pub const fn base_url(&self) -> &BaseUrl {
93        &self.base_url
94    }
95
96    /// The retry policy in force.
97    #[must_use]
98    pub const fn retry_policy(&self) -> &RetryPolicy {
99        &self.retry
100    }
101
102    /// Paces every call through this transport, and its clones, with `throttle`: a 429 or 529 on
103    /// one call then slows all of them. Share one [`Throttle`] between transports to pace them
104    /// together.
105    #[must_use]
106    pub fn with_throttle(mut self, throttle: Arc<Throttle>) -> Self {
107        self.throttle = Some(throttle);
108        self
109    }
110
111    /// Sends one kind of request until it succeeds, fails for good, or runs out of retries.
112    async fn call<T: DeserializeOwned>(
113        &self,
114        method: Method,
115        path: &str,
116        body: Option<Vec<u8>>,
117    ) -> Result<Reply<T>, Error> {
118        let url = self.base_url.endpoint(path).map_err(|error| {
119            Error::new(ErrorKind::InvalidRequest, error.to_string()).with_source(error)
120        })?;
121        if self.base_url.sends_key_in_clear() {
122            tracing::warn!(%url, "sending the API key over unencrypted HTTP");
123        }
124
125        let started = self.clock.now();
126        let mut retry = 0;
127        loop {
128            let attempt = retry + 1;
129            if let Some(throttle) = &self.throttle {
130                throttle.wait(self.clock.as_ref()).await;
131            }
132            let outcome = self
133                .attempt::<T>(&method, &url, body.as_deref(), attempt)
134                .await;
135
136            let failure = match outcome {
137                Ok((parsed, request_id, raw_body)) => {
138                    if let Some(throttle) = &self.throttle {
139                        throttle.speed_up();
140                    }
141                    let latency = self.clock.now().duration_since(started);
142                    let meta = ReplyMeta::new(request_id, latency, attempt);
143                    return Ok(Reply::new(parsed, meta).with_raw_body(raw_body));
144                }
145                Err(failure) => failure,
146            };
147
148            let delay = self
149                .retry
150                .delay(retry, failure.retry_after(), fastrand::f64());
151            if let (Some(throttle), ErrorKind::RateLimit | ErrorKind::Overloaded) =
152                (&self.throttle, failure.kind())
153            {
154                throttle.slow_down(self.clock.now(), delay);
155            }
156            if retry >= self.retry.max_retries || !self.should_retry(&failure) {
157                if failure.is_retryable() {
158                    tracing::debug!(attempt, "giving up: no retries left");
159                }
160                return Err(failure.with_attempts(attempt));
161            }
162            tracing::debug!(
163                attempt,
164                max_retries = self.retry.max_retries,
165                delay_ms = delay.as_millis(),
166                server_requested = failure.retry_after().is_some(),
167                reason = %failure_label(&failure),
168                "retrying"
169            );
170            self.clock.sleep(delay).await;
171            retry += 1;
172        }
173    }
174
175    /// Whether the policy retries this failure.
176    fn should_retry(&self, failure: &Error) -> bool {
177        match (failure.kind(), failure.status()) {
178            (ErrorKind::Timeout, _) => self.retry.retry_timeouts,
179            (ErrorKind::Connection, _) => self.retry.retry_connection_errors,
180            (_, Some(status)) => self.retry.retries_status(status),
181            _ => false,
182        }
183    }
184
185    /// One attempt: send, read, and either parse the body or describe the failure.
186    async fn attempt<T: DeserializeOwned>(
187        &self,
188        method: &Method,
189        url: &Url,
190        body: Option<&[u8]>,
191        attempt: u32,
192    ) -> Result<(T, Option<String>, String), Error> {
193        tracing::debug!(%method, %url, attempt, "sending request");
194        if let (true, Some(body)) = (self.log_bodies, body) {
195            let body = self.api_key.scrub(&String::from_utf8_lossy(body));
196            tracing::debug!(target: BODY_LOG_TARGET, %body, "request body");
197        }
198
199        let mut builder = self
200            .client
201            .request(method.clone(), url.clone())
202            .header(AUTHORIZATION, self.authorization.clone())
203            .header(ACCEPT, HeaderValue::from_static("application/json"));
204        if let Some(body) = body {
205            builder = builder
206                .header(CONTENT_TYPE, HeaderValue::from_static("application/json"))
207                .body(body.to_vec());
208        }
209
210        let sent = self.clock.now();
211        let response = builder
212            .send()
213            .await
214            .map_err(|error| self.transport_error(error))?;
215
216        let status = response.status();
217        let request_id = request_id(response.headers());
218        let retry_after = server_delay(response.headers(), self.clock.wall_time());
219        let bytes = read_capped(response).await.map_err(|error| match error {
220            ReadError::Transport(error) => self
221                .transport_error(error)
222                .with_request_id(request_id.clone()),
223            ReadError::TooLarge => Error::new(
224                ErrorKind::InvalidResponse,
225                format!("the response is larger than {MAX_RESPONSE_BYTES} bytes"),
226            )
227            .with_status(status.as_u16())
228            .with_request_id(request_id.clone()),
229        })?;
230        // A server that echoes the `Authorization` header back must not get the key into anything
231        // built from the body: an unknown answer kept as raw JSON, `--raw` output, an error or a log.
232        let bytes = self.api_key.scrub_bytes(bytes);
233
234        tracing::debug!(
235            %method,
236            %url,
237            status = status.as_u16(),
238            elapsed_ms = self.clock.now().duration_since(sent).as_millis(),
239            request_id = request_id.as_deref().unwrap_or("-"),
240            attempt,
241            "received response"
242        );
243        if self.log_bodies {
244            // Opting in to bodies is opting in to seeing `state`, never the key, which `bytes` no
245            // longer holds.
246            let body = String::from_utf8_lossy(&bytes);
247            tracing::debug!(target: BODY_LOG_TARGET, %body, "response body");
248        }
249
250        if !status.is_success() {
251            return Err(Error::from_response(status.as_u16(), &bytes, |text| {
252                self.api_key.scrub(text)
253            })
254            .with_request_id(request_id)
255            .with_retry_after(retry_after));
256        }
257        match serde_json::from_slice::<T>(&bytes) {
258            Ok(parsed) => Ok((
259                parsed,
260                request_id,
261                String::from_utf8_lossy(&bytes).into_owned(),
262            )),
263            Err(error) => Err(
264                Error::new(ErrorKind::InvalidResponse, describe_json_error(&error))
265                    .with_status(status.as_u16())
266                    .with_request_id(request_id),
267            ),
268        }
269    }
270
271    /// Describes a failure to send a request or to read its response.
272    fn transport_error(&self, error: reqwest::Error) -> Error {
273        let kind = if error.is_timeout() {
274            ErrorKind::Timeout
275        } else if error.is_builder() {
276            ErrorKind::InvalidRequest
277        } else {
278            ErrorKind::Connection
279        };
280        let message = match kind {
281            ErrorKind::Timeout => format!("no response within {:?}", self.retry.timeout),
282            _ => self.api_key.scrub(&root_cause(&error)),
283        };
284        // The URL is dropped from the source as a precaution; the trace output already has it.
285        Error::new(kind, message).with_source(error.without_url())
286    }
287}
288
289impl Transport for HttpTransport {
290    async fn evaluate(&self, request: &Request) -> Result<Reply<Response>, Error> {
291        let body = serde_json::to_vec(request).map_err(|error| {
292            Error::new(
293                ErrorKind::InvalidRequest,
294                format!("the request could not be serialised: {error}"),
295            )
296            .with_source(error)
297        })?;
298        self.call(Method::POST, "v1/systemone", Some(body)).await
299    }
300
301    async fn list_models(&self) -> Result<Reply<ModelList>, Error> {
302        self.call(Method::GET, "v1/models", None).await
303    }
304}
305
306impl fmt::Debug for HttpTransport {
307    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
308        formatter
309            .debug_struct("HttpTransport")
310            .field("base_url", &self.base_url)
311            .field("api_key", &self.api_key)
312            .field("retry", &self.retry)
313            .field("log_bodies", &self.log_bodies)
314            .field("throttle", &self.throttle)
315            .finish_non_exhaustive()
316    }
317}
318
319/// Configures and builds an [`HttpTransport`].
320pub struct HttpTransportBuilder {
321    api_key: ApiKey,
322    base_url: BaseUrl,
323    retry: RetryPolicy,
324    clock: Arc<dyn Clock>,
325    user_agent: Option<String>,
326    log_bodies: bool,
327}
328
329impl HttpTransportBuilder {
330    /// Sets the API root. The default is `https://api.typesafe.ai`.
331    #[must_use]
332    pub fn base_url(mut self, base_url: BaseUrl) -> Self {
333        self.base_url = base_url;
334        self
335    }
336
337    /// Sets the retry policy, which also holds the per-attempt timeout.
338    #[must_use]
339    pub fn retry_policy(mut self, retry: RetryPolicy) -> Self {
340        self.retry = retry;
341        self
342    }
343
344    /// Names the application for the `User-Agent` header, as `name/version`. It is placed in front
345    /// of this crate's own `jev-client/<version>`.
346    #[must_use]
347    pub fn user_agent(mut self, product: impl Into<String>) -> Self {
348        self.user_agent = Some(product.into());
349        self
350    }
351
352    /// Traces request and response bodies at `DEBUG` level, under the target
353    /// `jev_client::http::body`.
354    ///
355    /// Off by default, and meant to stay off: a request's `state` is often customer data, and an
356    /// error response can echo it back. The `Authorization` header is not traced either way.
357    #[must_use]
358    pub const fn log_bodies(mut self, enabled: bool) -> Self {
359        self.log_bodies = enabled;
360        self
361    }
362
363    /// Replaces the clock. Tests use this to observe retry delays without waiting for them.
364    #[must_use]
365    pub fn clock(mut self, clock: Arc<dyn Clock>) -> Self {
366        self.clock = clock;
367        self
368    }
369
370    /// Builds the transport.
371    ///
372    /// # Errors
373    ///
374    /// Fails with [`ErrorKind::InvalidRequest`] when the user agent is not a valid header value,
375    /// or when the TLS stack cannot be initialised on this system.
376    pub fn build(self) -> Result<HttpTransport, Error> {
377        let invalid = |what: &str, error: &dyn fmt::Display| {
378            Error::new(ErrorKind::InvalidRequest, format!("{what}: {error}"))
379        };
380
381        let own_agent = concat!("jev-client/", env!("CARGO_PKG_VERSION"));
382        let user_agent = self.user_agent.map_or_else(
383            || own_agent.to_owned(),
384            |product| format!("{product} {own_agent}"),
385        );
386        let user_agent = HeaderValue::from_str(&user_agent)
387            .map_err(|error| invalid("invalid user agent", &error))?;
388
389        let bearer = Zeroizing::new(format!("Bearer {}", self.api_key.expose()));
390        // `ApiKey` only admits visible ASCII, so this cannot fail; the error never holds the key.
391        let mut authorization = HeaderValue::from_str(&bearer).map_err(|_| {
392            Error::new(
393                ErrorKind::InvalidRequest,
394                "the API key cannot be sent in a header",
395            )
396        })?;
397        authorization.set_sensitive(true);
398
399        let provider = Arc::new(rustls::crypto::ring::default_provider());
400        let tls = rustls::ClientConfig::builder_with_provider(provider)
401            .with_safe_default_protocol_versions()
402            .map_err(|error| invalid("TLS could not be initialised", &error))?
403            .with_platform_verifier()
404            .map_err(|error| {
405                invalid(
406                    "the system certificate verifier could not be initialised",
407                    &error,
408                )
409            })?
410            .with_no_client_auth();
411
412        let client = reqwest::Client::builder()
413            .tls_backend_preconfigured(tls)
414            .https_only(self.base_url.is_https())
415            .redirect(reqwest::redirect::Policy::none())
416            .default_headers(HeaderMap::from_iter([(USER_AGENT, user_agent)]))
417            .timeout(self.retry.timeout)
418            .connect_timeout(CONNECT_TIMEOUT.min(self.retry.timeout))
419            .build()
420            .map_err(|error| invalid("the HTTP client could not be built", &error))?;
421
422        Ok(HttpTransport {
423            client,
424            base_url: self.base_url,
425            api_key: self.api_key,
426            authorization,
427            retry: self.retry,
428            clock: self.clock,
429            log_bodies: self.log_bodies,
430            throttle: None,
431        })
432    }
433}
434
435impl fmt::Debug for HttpTransportBuilder {
436    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
437        formatter
438            .debug_struct("HttpTransportBuilder")
439            .field("base_url", &self.base_url)
440            .field("api_key", &self.api_key)
441            .field("retry", &self.retry)
442            .field("user_agent", &self.user_agent)
443            .field("log_bodies", &self.log_bodies)
444            .finish_non_exhaustive()
445    }
446}
447
448/// A short label for a failure in trace output: the status when there is one, else the kind.
449fn failure_label(failure: &Error) -> String {
450    failure.status().map_or_else(
451        || format!("{:?}", failure.kind()),
452        |status| format!("HTTP {status}"),
453    )
454}
455
456fn request_id(headers: &HeaderMap) -> Option<String> {
457    let value = headers.get(REQUEST_ID_HEADER)?.to_str().ok()?.trim();
458    (!value.is_empty()).then(|| value.to_owned())
459}
460
461enum ReadError {
462    Transport(reqwest::Error),
463    TooLarge,
464}
465
466/// Reads a response body, refusing to buffer more than [`MAX_RESPONSE_BYTES`].
467async fn read_capped(mut response: reqwest::Response) -> Result<Vec<u8>, ReadError> {
468    if response
469        .content_length()
470        .is_some_and(|length| length > MAX_RESPONSE_BYTES as u64)
471    {
472        return Err(ReadError::TooLarge);
473    }
474    let mut body = Vec::new();
475    while let Some(chunk) = response.chunk().await.map_err(ReadError::Transport)? {
476        if body.len() + chunk.len() > MAX_RESPONSE_BYTES {
477            return Err(ReadError::TooLarge);
478        }
479        body.extend_from_slice(&chunk);
480    }
481    Ok(body)
482}
483
484/// Describes why a body failed to parse, by category and position only.
485///
486/// `serde_json`'s own message is not used, and the error is not kept as a source, because a type
487/// mismatch quotes the offending value (`invalid type: string "..."`) and a response can carry
488/// text that came from the request. Body logging is the way to see the content.
489fn describe_json_error(error: &serde_json::Error) -> String {
490    use serde_json::error::Category;
491
492    let problem = match error.classify() {
493        Category::Syntax => "the body is not valid JSON",
494        Category::Eof => "the body ends unexpectedly",
495        Category::Data => "the body is JSON but not in the documented shape",
496        Category::Io => "the body could not be read",
497    };
498    format!(
499        "{problem} (line {}, column {})",
500        error.line(),
501        error.column()
502    )
503}
504
505/// The innermost cause of an error, which is where `reqwest` keeps the useful part
506/// ("connection refused", "certificate has expired", ...).
507fn root_cause(error: &(dyn std::error::Error + 'static)) -> String {
508    let mut cause = error;
509    while let Some(next) = cause.source() {
510        cause = next;
511    }
512    cause.to_string()
513}