use super::{
client::{HttpClient, Pool},
config::{self, HttpMethod, SinkConfig},
metrics::ClientMetrics,
record::HttpPublish,
};
use crate::{
codec::Encoder,
error::{BoxError, Error, ensure},
sink::{PublishRejected, Sink},
};
use bytes::Bytes;
use http_body_util::{BodyExt, Full};
use hyper::{
HeaderMap, Method, Request, StatusCode, Uri,
body::Incoming,
header::{HeaderName, HeaderValue},
};
use std::{
collections::HashSet,
sync::{
Arc, OnceLock,
atomic::{AtomicBool, Ordering},
},
time::{Duration, Instant},
};
const ERROR_BODY_EXCERPT: usize = 512;
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct HttpPrepared {
headers: HeaderMap,
body: Bytes,
}
impl HttpPrepared {
pub fn body(&self) -> &[u8] {
&self.body
}
pub fn headers(&self) -> impl Iterator<Item = (&str, &[u8])> {
self.headers
.iter()
.map(|(name, value)| (name.as_str(), value.as_bytes()))
}
}
struct Target {
uri: Uri,
method: Method,
headers: HeaderMap,
timeout: Duration,
display: String,
metrics: ClientMetrics,
}
struct State {
target: OnceLock<Target>,
closed: AtomicBool,
}
impl State {
fn new() -> Arc<Self> {
Arc::new(Self {
target: OnceLock::new(),
closed: AtomicBool::new(false),
})
}
}
pub struct HttpSink<C> {
http: HttpClient,
config: SinkConfig,
codec: Arc<C>,
state: Arc<State>,
}
impl<C> Clone for HttpSink<C> {
fn clone(&self) -> Self {
Self {
http: self.http.clone(),
config: self.config.clone(),
codec: self.codec.clone(),
state: self.state.clone(),
}
}
}
impl<C> HttpSink<C> {
pub(super) fn new(http: HttpClient, config: SinkConfig, codec: C) -> Self {
Self {
http,
config,
codec: Arc::new(codec),
state: State::new(),
}
}
pub fn method(mut self, method: HttpMethod) -> Self {
self.config.method = method;
self.state = State::new();
self
}
pub fn header(mut self, name: impl Into<String>, value: impl Into<String>) -> Self {
self.config.headers.push((name.into(), value.into()));
self.state = State::new();
self
}
pub fn timeout(mut self, timeout: Duration) -> Self {
self.config.timeout = Some(timeout);
self.state = State::new();
self
}
pub fn validate(&self) -> Result<(), Error> {
self.http.validate()?;
self.config.validate()
}
fn target(&self) -> Result<&Target, Error> {
if let Some(target) = self.state.target.get() {
return Ok(target);
}
self.validate()?;
let uri = self.config.uri()?;
let mut headers = self.http.headers()?;
let own = config::headers(&self.config.headers)?;
for name in own.keys() {
headers.remove(name);
}
headers.extend(own);
let location = format!(
"{}://{}{}",
uri.scheme_str().unwrap_or_default(),
uri.authority().map(|a| a.as_str()).unwrap_or_default(),
uri.path()
);
let method = self.config.method.as_hyper();
let target = Target {
metrics: ClientMetrics::new(&location),
display: format!("{method} {location}"),
timeout: self.config.timeout.unwrap_or(self.http.default_timeout()),
uri,
method,
headers,
};
let _ = self.state.target.set(target);
Ok(self.state.target.get().expect("target was set"))
}
fn client(&self) -> Result<(Pool, &Target), BoxError> {
ensure!(
!self.state.closed.load(Ordering::Acquire),
Error::closed,
"HTTP sink is closed"
);
let target = self.target()?;
let pool = self.http.pool(target.uri.scheme_str() == Some("https"))?;
Ok((pool, target))
}
async fn send(
&self,
client: Pool,
target: &Target,
output: &HttpPrepared,
) -> Result<(), BoxError> {
let mut request = Request::new(Full::new(output.body.clone()));
*request.method_mut() = target.method.clone();
*request.uri_mut() = target.uri.clone();
*request.headers_mut() = output.headers.clone();
let started = Instant::now();
let exchange = async {
let response = client.request(request).await?;
let (parts, body) = response.into_parts();
let keep = if parts.status.is_success() {
0
} else {
ERROR_BODY_EXCERPT
};
Ok::<_, BoxError>((parts.status, read_body(body, keep).await))
};
let (status, body) = match tokio::time::timeout(target.timeout, exchange).await {
Ok(Ok(response)) => response,
Ok(Err(error)) => {
target.metrics.attempt("error", started);
return Err(Error::wrap(format!("HTTP {} failed", target.display), error).into());
}
Err(_) => {
target.metrics.attempt("timeout", started);
return Err(Error::msg(format!(
"HTTP {} timed out after {:?}",
target.display, target.timeout
))
.into());
}
};
target.metrics.attempt(status.as_str(), started);
if status.is_success() {
return Ok(());
}
let excerpt = String::from_utf8_lossy(&body);
let excerpt = excerpt.trim();
let error = if excerpt.is_empty() {
Error::msg(format!("HTTP {} answered {status}", target.display))
} else {
Error::msg(format!(
"HTTP {} answered {status}: {excerpt}",
target.display
))
};
if retryable(status) {
Err(error.into())
} else {
Err(PublishRejected::wrap(error))
}
}
}
impl<C> Sink<HttpPublish<C::Value>> for HttpSink<C>
where
C: Encoder<Value: Send + Sync + 'static>,
{
type Prepared = HttpPrepared;
fn prepare(&self, record: HttpPublish<C::Value>) -> Result<Self::Prepared, BoxError> {
let body = Bytes::from(self.codec.encode(&record.body)?);
let mut headers = self.target()?.headers.clone();
let mut replaced = HashSet::new();
for (name, value) in record.headers {
let name = HeaderName::from_bytes(name.as_bytes())
.map_err(|_| Error::invalid_record(format!("invalid HTTP header name {name:?}")))?;
let value = HeaderValue::from_bytes(&value).map_err(|_| {
Error::invalid_record(format!("invalid value for HTTP header {name:?}"))
})?;
if replaced.insert(name.clone()) {
headers.remove(&name);
}
headers.append(name, value);
}
Ok(HttpPrepared { headers, body })
}
async fn publish(&self, output: &Self::Prepared) -> Result<(), BoxError> {
let (client, target) = self.client()?;
self.send(client, target, output).await
}
async fn close(&self) -> Result<(), BoxError> {
self.state.closed.store(true, Ordering::Release);
Ok(())
}
}
fn retryable(status: StatusCode) -> bool {
status.is_server_error()
|| status == StatusCode::REQUEST_TIMEOUT
|| status == StatusCode::TOO_MANY_REQUESTS
}
async fn read_body(mut body: Incoming, keep: usize) -> Vec<u8> {
let mut kept = Vec::new();
while let Some(Ok(frame)) = body.frame().await {
if let Ok(data) = frame.into_data() {
let take = keep.saturating_sub(kept.len()).min(data.len());
kept.extend_from_slice(&data[..take]);
}
}
kept
}