use std::sync::Arc;
use std::time::Duration;
use reqwest::header::{HeaderMap, HeaderName, HeaderValue};
use serde::de::DeserializeOwned;
use url::Url;
use crate::auth::{AuthRequest, Authenticator};
use crate::clock::Clock;
use crate::error::{
ApiError, ConnectionError, Error, ErrorCategory, MAX_SERVER_WAIT, Result, error_from_response,
redirect_error,
};
use crate::limiter::{PlacementHold, RateLimiter};
use crate::operations_gen::OperationId;
use crate::signing::{encode_bytes, encode_query};
const BACKOFF_BASE: Duration = Duration::from_millis(500);
const BACKOFF_MAX: Duration = Duration::from_secs(10);
const MAX_RESPONSE_BODY: usize = 64 << 20;
pub(crate) const ORDER_ACTIONS_LIMIT: &str = "order_actions";
#[derive(Debug)]
pub struct RetryInfo<'a> {
pub operation: OperationId,
pub method: &'static str,
pub path: &'static str,
pub attempt: u32,
pub delay: Duration,
pub error: &'a Error,
pub idempotency_key: Option<&'a str>,
}
pub(crate) type RetryHook = Arc<dyn Fn(&RetryInfo<'_>) + Send + Sync>;
pub(crate) type Random = Arc<dyn Fn() -> f64 + Send + Sync>;
pub(crate) struct Call {
pub(crate) op: OperationId,
pub(crate) path: Vec<(&'static str, String)>,
pub(crate) query: Vec<(&'static str, String)>,
pub(crate) body: Option<Vec<u8>>,
pub(crate) text: bool,
pub(crate) idempotency_key: Option<String>,
pub(crate) no_idempotency_key: bool,
pub(crate) cost: u32,
pub(crate) order_actions: u32,
pub(crate) busy_not_retried: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum Attempt {
Ambiguous,
Refused,
}
impl Attempt {
pub(crate) fn of(e: &Error) -> Attempt {
match e {
Error::Connection(_) | Error::Decode(_) => Attempt::Ambiguous,
Error::Api(a) if a.status >= 500 && !matches!(a.status, 501 | 503) => {
Attempt::Ambiguous
}
_ => Attempt::Refused,
}
}
}
impl Call {
pub(crate) fn new(op: OperationId) -> Call {
Call {
op,
path: vec![],
query: vec![],
body: None,
text: false,
idempotency_key: None,
no_idempotency_key: false,
cost: 1,
order_actions: 0,
busy_not_retried: false,
}
}
pub(crate) fn busy_not_retried(mut self) -> Call {
self.busy_not_retried = true;
self
}
pub(crate) fn cost(mut self, tokens: u32) -> Call {
self.cost = tokens;
self
}
pub(crate) fn order_actions(mut self, n: u32) -> Call {
self.order_actions = n;
self
}
pub(crate) fn path(mut self, name: &'static str, value: &str) -> Call {
self.path.push((name, value.to_string()));
self
}
pub(crate) fn query(mut self, q: Vec<(&'static str, String)>) -> Call {
self.query = q;
self
}
pub(crate) fn json<T: serde::Serialize>(mut self, body: &T) -> Result<Call> {
let bytes = serde_json::to_vec(body).map_err(|e| {
Error::config(format!(
"{}: cannot encode the request body: {e}",
self.op.as_str()
))
})?;
self.body = Some(bytes);
Ok(self)
}
}
#[derive(Clone)]
pub(crate) struct Resolved {
pub(crate) timeout: Duration,
pub(crate) max_retries: u32,
pub(crate) idempotency_key: Option<String>,
}
pub(crate) struct Raw {
pub(crate) body: Vec<u8>,
}
pub(crate) struct Transport {
pub(crate) base_url: String,
pub(crate) origin: String,
pub(crate) http: reqwest::Client,
pub(crate) auth: Option<Arc<dyn Authenticator>>,
pub(crate) limiter: Option<RateLimiter>,
pub(crate) placement_hold: Option<PlacementHold>,
pub(crate) order_limiter: Option<RateLimiter>,
pub(crate) user_agent: String,
pub(crate) timeout: Duration,
pub(crate) max_retries: u32,
pub(crate) clock: Arc<dyn Clock>,
pub(crate) random: Random,
pub(crate) on_retry: Option<RetryHook>,
pub(crate) allow_insecure: bool,
}
impl Transport {
pub(crate) async fn request(&self, c: Call, o: &Resolved) -> Result<Raw> {
self.request_noting(c, o, &mut Vec::new()).await
}
pub(crate) async fn request_noting(
&self,
mut c: Call,
o: &Resolved,
attempts: &mut Vec<Attempt>,
) -> Result<Raw> {
let info = c.op.info();
if info.method != "GET" && c.idempotency_key.is_none() && !c.no_idempotency_key {
c.idempotency_key = Some(o.idempotency_key.clone().unwrap_or_else(new_id));
}
let max_retries = if repeat_safe(&c) { o.max_retries } else { 0 };
let mut attempt = 0;
loop {
match self.attempt(&c, o).await {
Ok(raw) => return Ok(raw),
Err(e) => {
attempts.push(Attempt::of(&e));
if attempt >= max_retries
|| !e.is_retryable()
|| (c.busy_not_retried && e.api().is_some_and(ApiError::is_futures_busy))
{
return Err(e);
}
self.backoff(c.op, attempt, &e, c.idempotency_key.as_deref())
.await;
attempt += 1;
}
}
}
}
pub(crate) async fn backoff(
&self,
op: OperationId,
attempt: u32,
cause: &Error,
idem: Option<&str>,
) {
let d = self.retry_delay(attempt, cause);
self.wait(op, attempt, cause, idem, d).await;
}
pub(crate) async fn wait(
&self,
op: OperationId,
attempt: u32,
cause: &Error,
idem: Option<&str>,
d: Duration,
) {
if let Some(hook) = &self.on_retry {
let info = op.info();
hook(&RetryInfo {
operation: op,
method: info.method,
path: info.path,
attempt: attempt + 1,
delay: d,
error: cause,
idempotency_key: idem,
});
}
self.clock.sleep(d).await;
}
pub(crate) fn retry_delay(&self, attempt: u32, cause: &Error) -> Duration {
if let Some(ra) = cause.api().and_then(|e| e.retry_after) {
let jitter = Duration::from_millis(((self.random)() * 250.0) as u64);
return ra.min(MAX_SERVER_WAIT).saturating_add(jitter);
}
let capped = BACKOFF_BASE
.saturating_mul(2u32.saturating_pow(attempt.min(20)))
.min(BACKOFF_MAX);
Duration::from_nanos(((self.random)() * capped.as_nanos() as f64).ceil() as u64)
}
pub(crate) async fn attempt(&self, c: &Call, o: &Resolved) -> Result<Raw> {
let mut e = match self.attempt_once(c, o).await {
Err(Error::Api(e)) => e,
other => return other,
};
if e.code.as_str() == "SIGNATURE_REQUIRED" {
e.message = "this API key must sign its requests: use AuthScheme::Hmac (the default) \
instead of AuthScheme::Headers"
.into();
e.retryable = false;
return Err(Error::Api(e));
}
if e.code.as_str() == "KEY_NOT_SIGNABLE" {
e.message =
"create a new API key; keys issued before request signing can't sign".into();
e.retryable = false;
return Err(Error::Api(e));
}
if e.code.as_str() != "SIGNATURE_EXPIRED" {
return Err(Error::Api(e));
}
let server_ms = e
.details
.get("server_time_ms")
.and_then(|v| v.as_i64().or_else(|| v.as_f64().map(|f| f as i64)));
let adjusted = match (&self.auth, server_ms) {
(Some(auth), Some(ms)) => auth.adjust_clock(ms),
_ => None,
};
match adjusted {
Some(true) => self.attempt_once(c, o).await,
Some(false) => {
e.message = "the local clock is more than 1 hour away from the server's: fix the system clock".into();
e.retryable = false;
Err(Error::Api(e))
}
None => Err(Error::Api(e)), }
}
async fn attempt_once(&self, c: &Call, o: &Resolved) -> Result<Raw> {
let info = c.op.info();
let url = self.build_url(c)?;
if info.auth == "api_key" && self.auth.is_none() {
return Err(Error::config(format!(
"{} {} needs an API key: create the client with api_key and api_secret",
info.method, info.path
)));
}
if let Some(l) = &self.limiter {
l.acquire(c.cost).await;
}
if c.order_actions > 0 {
if let Some(h) = &self.placement_hold {
h.wait().await;
}
if let Some(l) = &self.order_limiter {
l.acquire(c.order_actions).await;
}
}
let mut headers = HeaderMap::new();
let accept = if c.text {
"text/csv, application/json"
} else {
"application/json"
};
headers.insert("accept", HeaderValue::from_static(accept));
headers.insert(
"user-agent",
HeaderValue::from_str(&self.user_agent)
.map_err(|_| Error::config("invalid User-Agent"))?,
);
if c.body.is_some() {
headers.insert("content-type", HeaderValue::from_static("application/json"));
}
if info.method != "GET"
&& let Some(k) = &c.idempotency_key
{
headers.insert(
"idempotency-key",
HeaderValue::from_str(k).map_err(|_| Error::config("invalid Idempotency-Key"))?,
);
}
if info.auth == "api_key" {
if origin(&url) != self.origin {
return Err(Error::config(format!(
"{} {}: refusing to send credentials to {}",
info.method,
info.path,
origin(&url)
)));
}
let auth = self.auth.as_ref().expect("checked above");
let mut req = AuthRequest {
method: info.method,
url: &url,
body: c.body.as_deref(),
headers: vec![],
};
auth.authenticate(&mut req)?;
for (k, v) in req.headers {
let name = HeaderName::from_bytes(k.as_bytes())
.map_err(|_| Error::config("invalid header name"))?;
let mut value = HeaderValue::from_str(&v)
.map_err(|_| Error::config("invalid credential header value"))?;
value.set_sensitive(true);
headers.insert(name, value);
}
}
let method = reqwest::Method::from_bytes(info.method.as_bytes()).expect("valid method");
let mut rb = self
.http
.request(method, url)
.headers(headers)
.timeout(o.timeout);
if let Some(b) = &c.body {
rb = rb.body(b.clone());
}
let conn_err = |e: reqwest::Error| {
Error::Connection(ConnectionError {
method: info.method.to_string(),
path: info.path.to_string(),
timeout: e.is_timeout(),
message: self.redact(&e.to_string()),
})
};
let mut resp = rb.send().await.map_err(conn_err)?;
let status = resp.status().as_u16();
let resp_headers = resp.headers().clone();
let mut body = Vec::new();
while let Some(chunk) = resp.chunk().await.map_err(conn_err)? {
if body.len() + chunk.len() > MAX_RESPONSE_BODY {
return Err(Error::Decode(format!(
"{} {}: response larger than 64 MiB",
info.method, info.path
)));
}
body.extend_from_slice(&chunk);
}
if let Some(l) = &self.limiter {
l.update(&resp_headers);
}
let redact = |s: &str| self.redact(s);
if (300..=399).contains(&status) {
return Err(redirect_error(status, &resp_headers, &redact).into());
}
if !(200..=299).contains(&status) {
let e = error_from_response(status, &body, &resp_headers, &redact);
if e.is(ErrorCategory::RateLimited)
&& let (Some(ra), Some(l)) = (e.retry_after, &self.limiter)
{
let order_actions = e.details.get("limit").and_then(serde_json::Value::as_str)
== Some(ORDER_ACTIONS_LIMIT);
match &self.placement_hold {
Some(h) if order_actions => h.hold_for(ra),
_ => l.block_for(ra),
}
}
return Err(e.into());
}
if !c.text
&& !body.is_empty()
&& serde_json::from_slice::<serde::de::IgnoredAny>(&body).is_err()
{
let ct = resp_headers
.get("content-type")
.and_then(|v| v.to_str().ok())
.unwrap_or("unknown content");
return Err(Error::Decode(format!(
"{} {}: expected JSON, got {ct:?}",
info.method, info.path
)));
}
Ok(Raw { body })
}
pub(crate) fn build_url(&self, c: &Call) -> Result<Url> {
let info = c.op.info();
let mut path = String::new();
let mut rest = info.path;
while let Some(start) = rest.find('{') {
let end = rest[start..]
.find('}')
.map(|e| start + e)
.expect("well-formed path template");
path.push_str(&rest[..start]);
let name = &rest[start + 1..end];
let value = c
.path
.iter()
.find(|(n, _)| *n == name)
.map(|(_, v)| v.as_str())
.unwrap_or("");
if value.is_empty() || value.contains(['\r', '\n']) {
return Err(Error::config(format!(
"{} {}: {name} is required",
info.method, info.path
)));
}
if value == "." || value == ".." {
return Err(Error::config(format!(
"{} {}: {name} must not be \".\" or \"..\"",
info.method, info.path
)));
}
path.push_str(&encode_segment(value));
rest = &rest[end + 1..];
}
path.push_str(rest);
let mut url = Url::parse(&format!("{}{}", self.base_url, path)).map_err(|e| {
Error::config(format!("{} {}: invalid URL: {e}", info.method, info.path))
})?;
if !c.query.is_empty() {
url.set_query(Some(&encode_query(&c.query)));
}
Ok(url)
}
pub(crate) fn redact(&self, s: &str) -> String {
match &self.auth {
Some(a) => a.redact(s),
None => s.to_string(),
}
}
}
fn repeat_safe(c: &Call) -> bool {
c.op.info().method == "GET"
|| matches!(
c.op,
OperationId::CancelAll
| OperationId::CancelAllAfter
| OperationId::PlaceBatch
| OperationId::CancelBatch
| OperationId::CancelOrderByClientId
)
|| (matches!(c.op, OperationId::JoinPool | OperationId::ExitPool)
&& c.idempotency_key.is_some())
}
fn encode_segment(v: &str) -> String {
encode_bytes(v.as_bytes())
}
pub(crate) fn origin(u: &Url) -> String {
let mut s = format!(
"{}://{}",
u.scheme().to_lowercase(),
u.host_str().unwrap_or("").to_lowercase()
);
if let Some(p) = u.port() {
s.push_str(&format!(":{p}"));
}
s
}
#[derive(serde::Deserialize)]
struct Envelope<T> {
data: T,
}
pub(crate) fn decode_data<T: DeserializeOwned>(op: OperationId, raw: &Raw) -> Result<T> {
serde_json::from_slice::<Envelope<T>>(&raw.body)
.map(|e| e.data)
.map_err(|e| Error::Decode(format!("{}: cannot decode the response: {e}", op.as_str())))
}
pub(crate) fn decode<T: DeserializeOwned>(op: OperationId, raw: &Raw) -> Result<T> {
serde_json::from_slice::<T>(&raw.body)
.map_err(|e| Error::Decode(format!("{}: cannot decode the response: {e}", op.as_str())))
}
pub(crate) fn new_id() -> String {
uuid::Uuid::new_v4().to_string()
}