use std::{
error::{self},
fmt::Debug,
io,
marker::PhantomData,
time::SystemTime,
};
use bytes::Bytes;
use nanoid::nanoid;
use opentelemetry::KeyValue;
use rama::{Context, Layer, Service};
use tokio::{
io::{AsyncReadExt, AsyncWriteExt, BufWriter},
net::{TcpListener, TcpStream},
task::JoinSet,
};
use tokio_util::sync::CancellationToken;
use tracing::{debug, error, instrument};
use crate::{
BYTES_RECEIVED, BYTES_SENT, Error, REQUEST_DURATION, REQUEST_SIZE, RESPONSE_SIZE, frame_length,
};
#[derive(Clone, Debug, Default)]
pub struct TcpListenerLayer {
cancellation: CancellationToken,
}
impl TcpListenerLayer {
pub fn new(cancellation: CancellationToken) -> Self {
Self { cancellation }
}
}
impl<S> Layer<S> for TcpListenerLayer {
type Service = TcpListenerService<S>;
fn layer(&self, inner: S) -> Self::Service {
Self::Service {
cancellation: self.cancellation.clone(),
inner,
}
}
}
#[derive(Clone, Default)]
pub struct TcpListenerService<S> {
cancellation: CancellationToken,
inner: S,
}
impl<S> Debug for TcpListenerService<S> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct(stringify!(TcpListenerService)).finish()
}
}
impl<State, S> Service<State, TcpListener> for TcpListenerService<S>
where
S: Service<State, TcpStream> + Clone,
S::Response: Debug,
S::Error: error::Error,
State: Clone + Send + Sync + 'static,
{
type Response = ();
type Error = S::Error;
#[instrument(skip(ctx, req))]
async fn serve(
&self,
ctx: Context<State>,
req: TcpListener,
) -> Result<Self::Response, Self::Error> {
let mut set = JoinSet::new();
loop {
tokio::select! {
Ok((stream, addr)) = req.accept() => {
debug!(?req, ?stream, %addr);
let service = self.inner.clone();
let ctx = ctx.clone();
let handle = set.spawn(async move {
match service.serve(ctx, stream).await {
Err(error) => {
debug!(%addr, %error);
},
Ok(response) => {
debug!(%addr, ?response)
}
}
});
debug!(?handle);
continue;
}
v = set.join_next(), if !set.is_empty() => {
debug!(?v);
}
cancelled = self.cancellation.cancelled() => {
debug!(?cancelled);
break;
}
}
}
Ok(())
}
}
#[non_exhaustive]
#[derive(Clone, Debug, Default)]
pub struct TcpContext {
cluster_id: Option<String>,
maximum_frame_size: Option<usize>,
}
impl TcpContext {
pub fn cluster_id(self, cluster_id: Option<String>) -> Self {
Self { cluster_id, ..self }
}
pub fn maximum_frame_size(self, maximum_frame_size: Option<usize>) -> Self {
Self {
maximum_frame_size,
..self
}
}
}
#[derive(Clone, Debug, Default)]
pub struct TcpContextLayer {
state: TcpContext,
}
impl TcpContextLayer {
pub fn new(state: TcpContext) -> Self {
Self { state }
}
}
impl<S> Layer<S> for TcpContextLayer {
type Service = TcpContextService<S>;
fn layer(&self, inner: S) -> Self::Service {
Self::Service {
inner,
state: self.state.clone(),
}
}
}
#[derive(Clone)]
pub struct TcpContextService<S> {
inner: S,
state: TcpContext,
}
impl<S> Debug for TcpContextService<S> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct(stringify!(TcpContextService)).finish()
}
}
impl<State, S> Service<State, TcpStream> for TcpContextService<S>
where
S: Service<TcpContext, TcpStream>,
S::Error: From<io::Error>,
State: Clone + Send + Sync + 'static,
{
type Response = S::Response;
type Error = S::Error;
#[instrument(skip_all, fields(peer = %req.peer_addr()?))]
async fn serve(
&self,
ctx: Context<State>,
req: TcpStream,
) -> Result<Self::Response, Self::Error> {
let (ctx, _) = ctx.swap_state(self.state.clone());
self.inner.serve(ctx, req).await
}
}
#[derive(Clone, Copy, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)]
pub struct BytesTcpService;
impl Service<TcpStream, Bytes> for BytesTcpService {
type Response = Bytes;
type Error = Error;
#[instrument(skip(ctx, req))]
async fn serve(
&self,
mut ctx: Context<TcpStream>,
req: Bytes,
) -> Result<Self::Response, Self::Error> {
let stream = ctx.state_mut();
stream.write_all(&req[..]).await?;
BYTES_SENT.add(req.len() as u64, &[]);
let mut size = [0u8; 4];
_ = stream.read_exact(&mut size).await?;
let mut buffer: Vec<u8> = vec![0u8; frame_length(size)];
buffer[0..size.len()].copy_from_slice(&size[..]);
_ = stream.read_exact(&mut buffer[4..]).await?;
BYTES_RECEIVED.add(buffer.len() as u64, &[]);
Ok(Bytes::from(buffer))
}
}
#[derive(Clone, Copy, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)]
pub struct TcpBytesLayer<State = ()> {
_state: PhantomData<State>,
}
impl<S, State> Layer<S> for TcpBytesLayer<State> {
type Service = TcpBytesService<S, State>;
fn layer(&self, inner: S) -> Self::Service {
Self::Service {
inner,
_state: PhantomData,
}
}
}
#[derive(Clone, Copy, Default, Eq, Hash, Ord, PartialEq, PartialOrd)]
pub struct TcpBytesService<S, State> {
inner: S,
_state: PhantomData<State>,
}
impl<S, State> Debug for TcpBytesService<S, State> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct(stringify!(TcpBytesService)).finish()
}
}
impl<S, State> TcpBytesService<S, State> {
fn elapsed_millis(&self, start: SystemTime) -> u64 {
start
.elapsed()
.map_or(0, |duration| duration.as_millis() as u64)
}
}
impl<S, State> TcpBytesService<S, State>
where
S: Service<State, Bytes, Response = Bytes>,
S::Error: From<Error> + From<io::Error> + Debug,
State: Clone + Default + Send + Sync + 'static,
{
#[instrument(skip_all)]
async fn wait<R>(
&self,
req: &mut R,
maximum_frame_size: Option<usize>,
) -> Result<[u8; 4], S::Error>
where
R: AsyncReadExt + Unpin,
{
let mut size = [0u8; 4];
_ = req
.read_exact(&mut size)
.await
.inspect_err(|err| debug!(?err))?;
if maximum_frame_size
.is_some_and(|maximum_frame_size| maximum_frame_size > frame_length(size))
{
return Err(Into::into(Error::FrameTooBig(frame_length(size))));
} else {
Ok(size)
}
}
#[instrument(skip_all)]
async fn read<R>(&self, req: &mut R, size: [u8; 4]) -> Result<Bytes, S::Error>
where
R: AsyncReadExt + Unpin,
{
let mut request: Vec<u8> = vec![0u8; frame_length(size)];
request[0..size.len()].copy_from_slice(&size[..]);
_ = req
.read_exact(&mut request[4..])
.await
.inspect_err(|err| error!(?err))?;
BYTES_RECEIVED.add(request.len() as u64, &[]);
Ok(Bytes::from(request))
}
#[instrument(skip_all)]
async fn process(
&self,
attributes: &[KeyValue],
ctx: Context<TcpContext>,
request: Bytes,
) -> Result<Bytes, S::Error> {
REQUEST_SIZE.record(request.len() as u64, attributes);
let (ctx, _) = ctx.swap_state(State::default());
let request_start = SystemTime::now();
self.inner
.serve(ctx, request)
.await
.inspect_err(|err| error!(?err))
.inspect(|response| {
RESPONSE_SIZE.record(response.len() as u64, attributes);
let elapsed_millis = self.elapsed_millis(request_start);
REQUEST_DURATION.record(elapsed_millis, attributes);
})
}
#[instrument(skip_all)]
async fn write<W>(&self, req: &mut W, frame: Bytes) -> Result<(), S::Error>
where
W: AsyncWriteExt + Unpin,
{
let mut w = BufWriter::new(req);
w.write_all(&frame).await.inspect_err(|err| error!(?err))?;
BYTES_SENT.add(frame.len() as u64, &[]);
w.flush().await.map_err(Into::into)
}
#[instrument(skip_all, fields(id = nanoid!()))]
async fn req<R>(
&self,
req: &mut R,
maximum_frame_size: Option<usize>,
attributes: &[KeyValue],
ctx: Context<TcpContext>,
) -> Result<(), S::Error>
where
R: AsyncReadExt + AsyncWriteExt + Unpin,
{
let size = self.wait(req, maximum_frame_size).await?;
let request = self.read(req, size).await?;
let response = self.process(attributes, ctx, request).await?;
self.write(req, response).await
}
}
impl<S, State, Stream> Service<TcpContext, Stream> for TcpBytesService<S, State>
where
S: Service<State, Bytes, Response = Bytes>,
S::Error: From<Error> + From<io::Error> + Debug,
State: Clone + Default + Send + Sync + 'static,
Stream: AsyncReadExt + AsyncWriteExt + Unpin + Send + Sync + 'static,
{
type Response = ();
type Error = S::Error;
#[instrument(skip(ctx, req))]
async fn serve(
&self,
ctx: Context<TcpContext>,
mut req: Stream,
) -> Result<Self::Response, Self::Error> {
let attributes = {
let state = ctx.state();
let mut attributes = vec![];
if let Some(cluster_id) = state.cluster_id.clone() {
attributes.push(KeyValue::new("cluster_id", cluster_id))
}
attributes
};
let maximum_frame_size = ctx.state().maximum_frame_size;
loop {
let ctx = ctx.clone();
let attributes = attributes.clone();
self.req(&mut req, maximum_frame_size, &attributes[..], ctx)
.await?
}
}
}
#[derive(Clone, Copy, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)]
pub struct BytesLayer;
impl<S> Layer<S> for BytesLayer {
type Service = BytesService<S>;
fn layer(&self, inner: S) -> Self::Service {
Self::Service { inner }
}
}
#[derive(Clone, Copy, Default, Eq, Hash, Ord, PartialEq, PartialOrd)]
pub struct BytesService<S> {
inner: S,
}
impl<S> Debug for BytesService<S> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct(stringify!(BytesService)).finish()
}
}
impl<S, State> Service<State, Bytes> for BytesService<S>
where
S: Service<State, Bytes, Response = Bytes>,
State: Clone + Send + Sync + 'static,
{
type Response = Bytes;
type Error = S::Error;
#[instrument(skip_all)]
async fn serve(&self, ctx: Context<State>, req: Bytes) -> Result<Self::Response, Self::Error> {
debug!(req = ?&req[..]);
self.inner
.serve(ctx, req)
.await
.inspect(|response| debug!(response = ?&response[..]))
}
}