use std::collections::BTreeMap;
use std::fmt;
use std::panic::{catch_unwind, AssertUnwindSafe};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Once, RwLock};
use std::time::{Duration, SystemTime};
use bytes::Bytes;
use reqwest::header::{HeaderMap, HeaderName, HeaderValue};
use serde::de::DeserializeOwned;
use serde::Serialize;
use serde_json::Value;
use tokio::time::Instant;
use url::Url;
pub use reqwest::Method;
pub use tokio_util::sync::CancellationToken;
use super::config::{CommonConfig, TlsConfig};
use super::error::{Error, ErrorCode, Result};
use super::hooks::{guarded, ErrorInfo, RequestInfo, ResponseInfo};
use super::redact::{redact_headers, redact_text, redact_url, truncate};
use super::retry::parse_retry_after;
use super::secret::Secret;
pub const SDK_KEY: &str = "graylog";
pub const SDK_VERSION: &str = env!("CARGO_PKG_VERSION");
pub const REQUEST_ID_HEADERS: &[&str] = &[
"x-request-id",
"x-amzn-requestid",
"x-amz-request-id",
"x-datadog-request-id",
"request-id",
"x-correlation-id",
];
pub const MAX_VENDOR_MESSAGE: usize = 300;
const LOG_TARGET: &str = "grow_graylog::transport";
pub fn default_user_agent() -> String {
format!("grow-codes-{SDK_KEY}/{SDK_VERSION} (rust)")
}
static INSECURE_WARNING: Once = Once::new();
pub struct HttpTransport {
client: reqwest::Client,
config: CommonConfig,
user_agent: String,
known_secrets: RwLock<Vec<Secret>>,
closed: AtomicBool,
close_token: CancellationToken,
retry_after_hook: Option<RetryAfterHook>,
error_mapper: Option<ErrorMapper<'static>>,
}
type RetryAfterHook = Box<dyn Fn(u16, &HeaderMap) -> Option<Duration> + Send + Sync>;
type ErrorMapper<'a> = Box<dyn Fn(&ErrorResponse<'_>) -> Option<Error> + Send + Sync + 'a>;
#[derive(Debug)]
#[non_exhaustive]
pub struct ErrorResponse<'a> {
pub operation: &'a str,
pub method: &'a Method,
pub status: u16,
pub headers: &'a HeaderMap,
pub body: &'a [u8],
pub request_id: Option<&'a str>,
pub attempt: u32,
pub default_error: &'a Error,
}
impl ErrorResponse<'_> {
pub fn text(&self) -> Option<&str> {
std::str::from_utf8(self.body).ok()
}
pub fn json<T: DeserializeOwned>(&self) -> Option<T> {
serde_json::from_slice(self.body).ok()
}
pub fn vendor_text(&self) -> Option<String> {
vendor_message(self.body)
}
}
pub fn reqwest_client_builder(config: &CommonConfig) -> Result<reqwest::ClientBuilder> {
let builder = reqwest::Client::builder()
.no_proxy()
.connect_timeout(config.connect_timeout());
let mut builder = apply_tls(builder, config.tls())?;
if let Some(proxy) = config.proxy() {
let proxy = reqwest::Proxy::all(proxy)
.map_err(|e| Error::config("proxy is not a valid URL").with_source(e.without_url()))?;
builder = builder.proxy(proxy);
}
Ok(builder)
}
pub fn build_reqwest_client(config: &CommonConfig) -> Result<reqwest::Client> {
reqwest_client_builder(config)?
.build()
.map_err(|e| Error::config("cannot build the HTTP client").with_source(e.without_url()))
}
pub fn user_agent_for(config: &CommonConfig) -> String {
match config.user_agent_suffix() {
Some(suffix) if !suffix.is_empty() => format!("{} {suffix}", default_user_agent()),
_ => default_user_agent(),
}
}
pub(crate) fn apply_tls(
mut builder: reqwest::ClientBuilder,
tls: &TlsConfig,
) -> Result<reqwest::ClientBuilder> {
builder = builder.use_rustls_tls();
if tls.insecure_skip_verify {
INSECURE_WARNING.call_once(|| {
::tracing::warn!(
target: LOG_TARGET,
"TLS certificate verification is disabled (insecure_skip_verify); use only in tests"
);
});
builder = builder.danger_accept_invalid_certs(true);
}
let mut ca_pem = Vec::new();
if let Some(path) = &tls.ca_cert_path {
ca_pem.extend(read_file(path, "CA certificate")?);
ca_pem.push(b'\n');
}
if let Some(pem) = &tls.ca_cert_pem {
ca_pem.extend_from_slice(pem.as_bytes());
}
if !ca_pem.is_empty() {
let certs = reqwest::Certificate::from_pem_bundle(&ca_pem)
.map_err(|e| Error::config("CA certificate is not valid PEM").with_source(e))?;
if certs.is_empty() {
return Err(Error::config("CA certificate contains no certificate"));
}
for cert in certs {
builder = builder.add_root_certificate(cert);
}
}
if let (Some(cert), Some(key)) = (&tls.client_cert_path, &tls.client_key_path) {
let mut pem = read_file(cert, "client certificate")?;
pem.push(b'\n');
pem.extend(read_file(key, "client key")?);
let identity = reqwest::Identity::from_pem(&pem).map_err(|e| {
Error::config("client certificate or key is not valid PEM").with_source(e)
})?;
builder = builder.identity(identity);
}
Ok(builder)
}
impl fmt::Debug for HttpTransport {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("HttpTransport")
.field("base_url", &redact_url(self.config.url().as_str()))
.field("user_agent", &self.user_agent)
.field("closed", &self.is_closed())
.finish_non_exhaustive()
}
}
impl HttpTransport {
pub fn new(config: &CommonConfig) -> Result<Self> {
let client = build_reqwest_client(config)?;
Ok(HttpTransport {
client,
config: config.clone(),
user_agent: user_agent_for(config),
known_secrets: RwLock::new(Vec::new()),
closed: AtomicBool::new(false),
close_token: CancellationToken::new(),
retry_after_hook: None,
error_mapper: None,
})
}
pub fn with_error_mapper<F>(mut self, mapper: F) -> Self
where
F: Fn(&ErrorResponse<'_>) -> Option<Error> + Send + Sync + 'static,
{
self.error_mapper = Some(Box::new(mapper));
self
}
pub fn with_retry_after_hook<F>(mut self, hook: F) -> Self
where
F: Fn(u16, &HeaderMap) -> Option<Duration> + Send + Sync + 'static,
{
self.retry_after_hook = Some(Box::new(hook));
self
}
pub fn reqwest_client(&self) -> &reqwest::Client {
&self.client
}
pub fn url_for(&self, path: &str) -> Result<Url> {
self.join(path, &[])
}
pub fn ensure_open(&self, operation: &str) -> Result<()> {
if self.is_closed() {
return Err(Error::cancelled(format!("{operation}: client is closed")));
}
Ok(())
}
pub fn config(&self) -> &CommonConfig {
&self.config
}
pub fn user_agent(&self) -> &str {
&self.user_agent
}
pub fn add_known_secret(&self, secret: &Secret) {
if secret.expose_secret().chars().count() >= 4 {
self.known_secrets
.write()
.unwrap_or_else(|e| e.into_inner())
.push(secret.clone());
}
}
pub fn redact(&self, text: &str) -> String {
let secrets = self.known_secrets.read().unwrap_or_else(|e| e.into_inner());
let values: Vec<&str> = secrets.iter().map(Secret::expose_secret).collect();
redact_text(text, &values)
}
pub fn request(&self, method: Method, path: &str) -> RequestBuilder<'_> {
RequestBuilder {
transport: self,
method,
path: path.to_owned(),
query: Vec::new(),
headers: Vec::new(),
body: None,
idempotent: None,
operation: None,
cancel: None,
ok_statuses: Vec::new(),
secrets: Vec::new(),
error: None,
no_retry: false,
error_mapper: None,
}
}
pub fn get(&self, path: &str) -> RequestBuilder<'_> {
self.request(Method::GET, path)
}
pub fn post(&self, path: &str) -> RequestBuilder<'_> {
self.request(Method::POST, path)
}
pub fn put(&self, path: &str) -> RequestBuilder<'_> {
self.request(Method::PUT, path)
}
pub fn delete(&self, path: &str) -> RequestBuilder<'_> {
self.request(Method::DELETE, path)
}
pub fn close(&self) {
if !self.closed.swap(true, Ordering::SeqCst) {
self.close_token.cancel();
}
}
pub fn is_closed(&self) -> bool {
self.closed.load(Ordering::SeqCst)
}
fn join(&self, path: &str, query: &[(String, String)]) -> Result<Url> {
let mut url = if path.starts_with("http://") || path.starts_with("https://") {
Url::parse(path).map_err(|_| Error::validation("request URL is not valid"))?
} else {
let mut url = self.config.url().clone();
let (p, q) = match path.split_once('?') {
Some((p, q)) => (p, Some(q)),
None => (path, None),
};
let base = url.path().trim_end_matches('/').to_owned();
url.set_path(&format!("{base}/{}", p.trim_start_matches('/')));
url.set_query(q);
url
};
if !query.is_empty() {
url.query_pairs_mut().extend_pairs(query);
}
Ok(url)
}
fn header_map(&self, req: &RequestBuilder<'_>) -> Result<HeaderMap> {
let mut map = HeaderMap::new();
let mut put = |name: &str, value: &str| -> Result<()> {
let n = HeaderName::from_bytes(name.as_bytes())
.map_err(|_| Error::validation(format!("invalid header name {name:?}")))?;
let v = HeaderValue::from_str(value)
.map_err(|_| Error::validation(format!("invalid value for header {name:?}")))?;
map.insert(n, v);
Ok(())
};
put("user-agent", &self.user_agent)?;
put("accept", "application/json")?;
for (k, v) in self.config.headers() {
put(k, v)?;
}
if let Some((_, content_type)) = &req.body {
put("content-type", content_type)?;
}
for (k, v) in &req.headers {
put(k, v.expose_secret())?;
}
Ok(map)
}
async fn execute(&self, req: RequestBuilder<'_>) -> Result<Response> {
let operation = req.operation.clone().unwrap_or_else(|| {
format!(
"{} {}",
req.method,
req.path.split('?').next().unwrap_or("")
)
});
if let Some(err) = req.error.clone() {
return Err(err.with_message(format!("{operation}: invalid request body")));
}
if self.is_closed() {
return Err(Error::cancelled(format!("{operation}: client is closed")));
}
for s in &req.secrets {
self.add_known_secret(s);
}
let url = self.join(&req.path, &req.query)?;
let headers = self.header_map(&req)?;
let redacted_url = self.redact(&redact_url(url.as_str()));
let redacted_headers: BTreeMap<String, String> = redact_headers(
headers
.iter()
.map(|(k, v)| (k.as_str(), v.to_str().unwrap_or("[binary]"))),
)
.into_iter()
.map(|(k, v)| (k, self.redact(&v)))
.collect();
let method = req.method.clone();
let idempotent = req.idempotent.unwrap_or(matches!(
method,
Method::GET | Method::HEAD | Method::OPTIONS | Method::PUT | Method::DELETE
));
let mut policy = self.config.retry().clone();
if req.no_retry {
policy.max_retries = 0;
}
let request_timeout = self.config.request_timeout();
let deadline = Instant::now() + self.config.total_timeout();
let hooks = self.config.hooks().cloned();
let mut attempt: u32 = 0;
loop {
attempt += 1;
let retry_index = attempt - 1;
if let Some(h) = &hooks {
let info = RequestInfo {
method: method.to_string(),
url: redacted_url.clone(),
headers: redacted_headers.clone(),
attempt,
};
guarded(|| h.on_request(&info));
}
::tracing::debug!(target: LOG_TARGET, method = %method, url = %redacted_url, attempt, "request");
let remaining = deadline.saturating_duration_since(Instant::now());
let total_limited = remaining < request_timeout;
let attempt_timeout = request_timeout.min(remaining);
let mut builder = self
.client
.request(method.clone(), url.clone())
.headers(headers.clone());
if let Some((body, _)) = &req.body {
builder = builder.body(body.clone());
}
let started = Instant::now();
let fut = async move {
let resp = builder.send().await?;
let status = resp.status().as_u16();
let headers = resp.headers().clone();
let body = resp.bytes().await?;
Ok::<_, reqwest::Error>((status, headers, body))
};
let outcome = tokio::select! {
biased;
_ = wait_cancel(req.cancel.as_ref()) => None,
_ = self.close_token.cancelled() => None,
r = tokio::time::timeout(attempt_timeout, fut) => Some(r),
};
let Some(outcome) = outcome else {
let err = Error::cancelled(format!("{operation}: cancelled"));
self.emit_error(&hooks, &method, &redacted_url, &err, attempt, None);
return Err(err);
};
let mut retry_after = None;
let (err, may_retry) = match outcome {
Err(_elapsed) => {
let err = if total_limited {
Error::timeout(format!(
"{operation}: total timeout of {} ms exceeded",
self.config.total_timeout().as_millis()
))
} else {
Error::timeout(format!(
"{operation}: request timed out after {} ms",
request_timeout.as_millis()
))
};
(err, idempotent && !total_limited)
}
Ok(Err(e)) => self.network_error(&operation, e, idempotent),
Ok(Ok((status, resp_headers, body))) => {
let request_id = request_id(&resp_headers);
if let Some(h) = &hooks {
let info = ResponseInfo {
method: method.to_string(),
url: redacted_url.clone(),
status,
duration: started.elapsed(),
attempt,
request_id: request_id.clone(),
};
guarded(|| h.on_response(&info));
}
::tracing::debug!(target: LOG_TARGET, method = %method, url = %redacted_url, status, attempt, "response");
if (200..400).contains(&status) || req.ok_statuses.contains(&status) {
return Ok(Response {
status,
headers: resp_headers,
body,
request_id,
});
}
if let Some(hook) = &self.retry_after_hook {
retry_after =
catch_unwind(AssertUnwindSafe(|| hook(status, &resp_headers)))
.ok()
.flatten();
}
if retry_after.is_none() && matches!(status, 429 | 503) {
retry_after = resp_headers
.get("retry-after")
.and_then(|v| v.to_str().ok())
.and_then(|v| parse_retry_after(v, SystemTime::now()));
}
let err = self.status_error(&operation, status, &body, request_id.clone());
let retryable = match status {
429 => true,
408 | 502 | 503 | 504 => idempotent,
500 => idempotent,
_ => false,
};
let response = ErrorResponse {
operation: &operation,
method: &method,
status,
headers: &resp_headers,
body: &body,
request_id: request_id.as_deref(),
attempt,
default_error: &err,
};
match self.map_error(&req, &response) {
Some(mapped) => {
let retryable = retryable && mapped.is_retryable();
(mapped, retryable)
}
None => (err, retryable),
}
}
};
let err = match retry_after {
Some(ra) => err.with_retry_after(ra),
None => err,
};
if !may_retry || retry_index >= policy.max_retries {
self.emit_error(&hooks, &method, &redacted_url, &err, attempt, None);
return Err(err);
}
let mut delay = policy.backoff(retry_index);
if let Some(ra) = retry_after {
if ra > policy.max_retry_after {
self.emit_error(&hooks, &method, &redacted_url, &err, attempt, None);
return Err(err);
}
delay = delay.max(ra);
}
if Instant::now() + delay >= deadline {
self.emit_error(&hooks, &method, &redacted_url, &err, attempt, None);
if retry_after.is_some() {
return Err(err);
}
return Err(Error::timeout(format!(
"{operation}: total timeout of {} ms exhausted after {attempt} attempts",
self.config.total_timeout().as_millis()
))
.with_source(err));
}
self.emit_error(&hooks, &method, &redacted_url, &err, attempt, Some(delay));
let cancelled = tokio::select! {
biased;
_ = wait_cancel(req.cancel.as_ref()) => true,
_ = self.close_token.cancelled() => true,
_ = tokio::time::sleep(delay) => false,
};
if cancelled {
return Err(Error::cancelled(format!(
"{operation}: cancelled during retry backoff"
)));
}
}
}
fn map_error(&self, req: &RequestBuilder<'_>, response: &ErrorResponse<'_>) -> Option<Error> {
let mapped = [req.error_mapper.as_ref(), self.error_mapper.as_ref()]
.into_iter()
.flatten()
.find_map(|m| {
catch_unwind(AssertUnwindSafe(|| m(response)))
.ok()
.flatten()
})?;
let message = self.redact(mapped.message());
let mut mapped = mapped.with_message(message);
if mapped.http_status().is_none() {
mapped = mapped.with_status(response.status);
}
if let (None, Some(id)) = (mapped.request_id(), response.request_id) {
mapped = mapped.with_request_id(id);
}
Some(mapped)
}
fn emit_error(
&self,
hooks: &Option<std::sync::Arc<dyn super::hooks::Hooks>>,
method: &Method,
url: &str,
err: &Error,
attempt: u32,
retry_delay: Option<Duration>,
) {
::tracing::debug!(
target: LOG_TARGET,
method = %method,
url = %url,
attempt,
code = %err.code(),
error = %err,
will_retry = retry_delay.is_some(),
"request failed"
);
if let Some(h) = hooks {
let info = ErrorInfo {
method: method.to_string(),
url: url.to_owned(),
error: err.clone(),
attempt,
will_retry: retry_delay.is_some(),
retry_delay,
};
guarded(|| h.on_error(&info));
}
}
fn network_error(&self, operation: &str, e: reqwest::Error, idempotent: bool) -> (Error, bool) {
let before_send = e.is_connect();
let err = if e.is_timeout() {
Error::timeout(format!("{operation}: connection timed out"))
} else if before_send {
Error::unavailable(format!("{operation}: connection failed"))
} else if e.is_decode() || e.is_body() {
Error::unavailable(format!("{operation}: response could not be read"))
} else {
Error::unavailable(format!("{operation}: request failed"))
};
(err.with_source(e.without_url()), before_send || idempotent)
}
fn status_error(
&self,
operation: &str,
status: u16,
body: &[u8],
request_id: Option<String>,
) -> Error {
let code = ErrorCode::from_http_status(status).unwrap_or(ErrorCode::Unavailable);
let reason = match vendor_message(body) {
Some(text) => {
let text = truncate(&self.redact(&text), MAX_VENDOR_MESSAGE);
format!("{}: {text}", code.reason())
}
None => code.reason().to_owned(),
};
let err = Error::from_status(status, format!("{operation}: {reason} (HTTP {status})"));
match request_id {
Some(id) => err.with_request_id(id),
None => err,
}
}
}
impl Drop for HttpTransport {
fn drop(&mut self) {
self.close_token.cancel();
}
}
fn read_file(path: &std::path::Path, what: &str) -> Result<Vec<u8>> {
std::fs::read(path).map_err(|e| {
Error::config(format!("{what} file {} cannot be read", path.display())).with_source(e)
})
}
async fn wait_cancel(token: Option<&CancellationToken>) {
match token {
Some(t) => t.cancelled().await,
None => std::future::pending().await,
}
}
fn request_id(headers: &HeaderMap) -> Option<String> {
REQUEST_ID_HEADERS
.iter()
.find_map(|h| headers.get(*h).and_then(|v| v.to_str().ok()))
.filter(|v| !v.is_empty())
.map(str::to_owned)
}
fn vendor_message(body: &[u8]) -> Option<String> {
if body.is_empty() {
return None;
}
if let Ok(value) = serde_json::from_slice::<Value>(body) {
return json_message(&value);
}
let text = std::str::from_utf8(body).ok()?.trim();
if text.is_empty() || text.starts_with('<') {
return None;
}
text.lines().next().map(|line| line.trim().to_owned())
}
fn json_message(value: &Value) -> Option<String> {
match value {
Value::String(s) if !s.trim().is_empty() => Some(s.trim().to_owned()),
Value::Array(items) => items.iter().find_map(json_message),
Value::Object(map) => [
"message",
"error_description",
"error",
"errors",
"detail",
"title",
"msg",
"reason",
]
.iter()
.find_map(|k| map.get(*k).and_then(json_message)),
_ => None,
}
}
#[must_use = "a request does nothing until `send` is awaited"]
pub struct RequestBuilder<'a> {
transport: &'a HttpTransport,
method: Method,
path: String,
query: Vec<(String, String)>,
headers: Vec<(String, Secret)>,
body: Option<(Bytes, String)>,
idempotent: Option<bool>,
operation: Option<String>,
cancel: Option<CancellationToken>,
ok_statuses: Vec<u16>,
secrets: Vec<Secret>,
error: Option<Error>,
no_retry: bool,
error_mapper: Option<ErrorMapper<'a>>,
}
impl fmt::Debug for RequestBuilder<'_> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("RequestBuilder")
.field("method", &self.method)
.field("path", &self.path)
.finish_non_exhaustive()
}
}
impl<'a> RequestBuilder<'a> {
pub fn query(mut self, name: impl Into<String>, value: impl Into<String>) -> Self {
self.query.push((name.into(), value.into()));
self
}
pub fn header(mut self, name: impl Into<String>, value: impl Into<String>) -> Self {
self.headers.push((name.into(), Secret::new(value)));
self
}
pub fn secret_header(mut self, name: impl Into<String>, value: &Secret) -> Self {
self.secrets.push(value.clone());
self.headers.push((name.into(), value.clone()));
self
}
pub fn bearer(self, token: &Secret) -> Self {
let header = Secret::new(format!("Bearer {}", token.expose_secret()));
let mut this = self.secret_header("authorization", &header);
this.secrets.push(token.clone());
this
}
pub fn json<T: Serialize + ?Sized>(mut self, body: &T) -> Self {
match serde_json::to_vec(body) {
Ok(bytes) => self.body = Some((Bytes::from(bytes), "application/json".to_owned())),
Err(e) => self.error = Some(Error::from(e)),
}
self
}
pub fn body(mut self, body: impl Into<Bytes>, content_type: impl Into<String>) -> Self {
self.body = Some((body.into(), content_type.into()));
self
}
pub fn idempotent(mut self, idempotent: bool) -> Self {
self.idempotent = Some(idempotent);
self
}
pub fn operation(mut self, operation: impl Into<String>) -> Self {
self.operation = Some(operation.into());
self
}
pub fn cancel_token(mut self, token: CancellationToken) -> Self {
self.cancel = Some(token);
self
}
pub fn no_retry(mut self) -> Self {
self.no_retry = true;
self
}
pub fn map_error<F>(mut self, mapper: F) -> Self
where
F: Fn(&ErrorResponse<'_>) -> Option<Error> + Send + Sync + 'a,
{
self.error_mapper = Some(Box::new(mapper));
self
}
pub fn ok_status(mut self, status: u16) -> Self {
self.ok_statuses.push(status);
self
}
pub async fn send(self) -> Result<Response> {
let transport = self.transport;
transport.execute(self).await
}
}
#[derive(Debug, Clone)]
pub struct Response {
status: u16,
headers: HeaderMap,
body: Bytes,
request_id: Option<String>,
}
impl Response {
pub fn status(&self) -> u16 {
self.status
}
pub fn headers(&self) -> &HeaderMap {
&self.headers
}
pub fn header(&self, name: &str) -> Option<&str> {
self.headers.get(name).and_then(|v| v.to_str().ok())
}
pub fn request_id(&self) -> Option<&str> {
self.request_id.as_deref()
}
pub fn bytes(&self) -> &Bytes {
&self.body
}
pub fn text(&self) -> Result<String> {
String::from_utf8(self.body.to_vec())
.map_err(|_| Error::validation("response body is not valid UTF-8"))
}
pub fn json<T: DeserializeOwned>(&self) -> Result<T> {
Ok(serde_json::from_slice(&self.body)?)
}
}