use std::collections::VecDeque;
use std::io::Write;
use std::pin::Pin;
use std::task::{Context, Poll};
use bytes::Bytes;
use hyper::body::Frame;
use hyper::header::{
ACCEPT_ENCODING, ACCEPT_RANGES, CACHE_CONTROL, CONTENT_ENCODING, CONTENT_LENGTH, CONTENT_TYPE,
ETAG, HeaderMap, HeaderValue, VARY,
};
use hyper::{Response, StatusCode};
use plecto_control::{CompressionAlgorithm, CompressionConfig, RouteInfo};
use crate::{BoxError, ResponseBody};
const GZIP_LEVEL: u32 = 5;
const BROTLI_QUALITY: u32 = 4;
const BROTLI_LGWIN: u32 = 22;
const ZSTD_LEVEL: i32 = 3;
const ZSTD_WINDOW_LOG: u32 = 23;
const BROTLI_BUFFER: usize = 4096;
pub(crate) fn apply(
resp: Response<ResponseBody>,
route: &RouteInfo,
request: &hyper::http::request::Parts,
) -> Response<ResponseBody> {
let Some(cfg) = route.compression.as_deref() else {
return resp;
};
if request.method == hyper::Method::HEAD {
return resp;
}
compress_response(resp, &request.headers, cfg)
}
fn compress_response(
resp: Response<ResponseBody>,
request_headers: &HeaderMap,
cfg: &CompressionConfig,
) -> Response<ResponseBody> {
let (mut parts, body) = resp.into_parts();
if !response_eligible(parts.status, &parts.headers, cfg) {
return Response::from_parts(parts, body);
}
add_vary_accept_encoding(&mut parts.headers);
let Some(algo) = negotiate(request_headers, cfg.algorithms()) else {
return Response::from_parts(parts, body);
};
let Ok(encoder) = Encoder::new(algo) else {
return Response::from_parts(parts, body);
};
mark_compressed(&mut parts.headers, algo);
let compressed = CompressBody {
inner: body,
encoder: Some(parking_lot::Mutex::new(encoder)),
queue: VecDeque::new(),
};
Response::from_parts(parts, http_body_util::BodyExt::boxed(compressed))
}
fn negotiate(
request_headers: &HeaderMap,
algorithms: &[CompressionAlgorithm],
) -> Option<CompressionAlgorithm> {
let mut explicit = CodingWeights::default();
let mut star: Option<u16> = None;
let mut identity_q: Option<u16> = None;
let mut saw_header = false;
for value in request_headers.get_all(ACCEPT_ENCODING) {
saw_header = true;
let Ok(s) = value.to_str() else { continue };
for member in s.split(',') {
let mut params = member.split(';');
let coding = params.next().unwrap_or("").trim();
if coding.is_empty() {
continue;
}
let mut q: u16 = 1000;
let mut malformed = false;
for param in params {
let Some((name, val)) = param.split_once('=') else {
malformed = true;
break;
};
if name.trim().eq_ignore_ascii_case("q") {
match parse_qvalue(val.trim()) {
Some(v) => q = v,
None => {
malformed = true;
break;
}
}
}
}
if malformed {
continue;
}
if coding == "*" {
star = Some(q);
} else if coding.eq_ignore_ascii_case("identity") {
identity_q = Some(q);
} else if let Some(algo) = coding_algo(coding) {
*explicit.slot(algo) = Some(q);
}
}
}
if !saw_header {
return None;
}
let mut best: Option<(u16, CompressionAlgorithm)> = None;
for algo in algorithms {
let q = explicit.slot(*algo).or(star).unwrap_or(0);
if q == 0 {
continue;
}
if best.is_none_or(|(bq, _)| q > bq) {
best = Some((q, *algo));
}
}
let (bq, algo) = best?;
if identity_q.is_some_and(|iq| bq < iq) {
return None;
}
Some(algo)
}
#[derive(Default)]
struct CodingWeights {
zstd: Option<u16>,
br: Option<u16>,
gzip: Option<u16>,
}
impl CodingWeights {
fn slot(&mut self, algo: CompressionAlgorithm) -> &mut Option<u16> {
match algo {
CompressionAlgorithm::Zstd => &mut self.zstd,
CompressionAlgorithm::Br => &mut self.br,
CompressionAlgorithm::Gzip => &mut self.gzip,
}
}
}
fn coding_algo(token: &str) -> Option<CompressionAlgorithm> {
if token.eq_ignore_ascii_case("zstd") {
Some(CompressionAlgorithm::Zstd)
} else if token.eq_ignore_ascii_case("br") {
Some(CompressionAlgorithm::Br)
} else if token.eq_ignore_ascii_case("gzip") || token.eq_ignore_ascii_case("x-gzip") {
Some(CompressionAlgorithm::Gzip)
} else {
None
}
}
fn parse_qvalue(s: &str) -> Option<u16> {
let (int, frac) = s.split_once('.').unwrap_or((s, ""));
if frac.len() > 3 || !frac.bytes().all(|b| b.is_ascii_digit()) {
return None;
}
match int {
"0" => {
let mut thousandths: u16 = 0;
let mut place: u16 = 100;
for b in frac.bytes() {
thousandths += u16::from(b - b'0') * place;
place /= 10;
}
Some(thousandths)
}
"1" => frac.bytes().all(|b| b == b'0').then_some(1000),
_ => None,
}
}
fn response_eligible(status: StatusCode, headers: &HeaderMap, cfg: &CompressionConfig) -> bool {
if status.is_informational() || matches!(status.as_u16(), 204 | 206 | 304) {
return false;
}
if headers.contains_key(CONTENT_ENCODING) {
return false;
}
if cache_control_no_transform(headers) {
return false;
}
let essence_eligible = headers
.get(CONTENT_TYPE)
.and_then(|v| v.to_str().ok())
.map(|s| s.split(';').next().unwrap_or(s))
.is_some_and(|essence| cfg.content_type_eligible(essence));
if !essence_eligible {
return false;
}
match headers.get(CONTENT_LENGTH) {
Some(v) => v
.to_str()
.ok()
.and_then(|s| s.trim().parse::<u64>().ok())
.is_some_and(|len| len >= cfg.min_length()),
None => true,
}
}
fn cache_control_no_transform(headers: &HeaderMap) -> bool {
headers.get_all(CACHE_CONTROL).iter().any(|v| {
v.to_str().is_ok_and(|s| {
s.split(',').any(|directive| {
directive
.split('=')
.next()
.is_some_and(|name| name.trim().eq_ignore_ascii_case("no-transform"))
})
})
})
}
fn add_vary_accept_encoding(headers: &mut HeaderMap) {
let already = headers.get_all(VARY).iter().any(|v| {
v.to_str().is_ok_and(|s| {
s.split(',').any(|member| {
let member = member.trim();
member == "*" || member.eq_ignore_ascii_case("accept-encoding")
})
})
});
if !already {
headers.append(VARY, HeaderValue::from_static("accept-encoding"));
}
}
fn mark_compressed(headers: &mut HeaderMap, algo: CompressionAlgorithm) {
headers.insert(CONTENT_ENCODING, HeaderValue::from_static(algo.token()));
headers.remove(CONTENT_LENGTH);
headers.remove(ACCEPT_RANGES);
if let Some(etag) = headers.get(ETAG) {
let bytes = etag.as_bytes();
if !bytes.starts_with(b"W/") {
let mut weak = Vec::with_capacity(bytes.len() + 2);
weak.extend_from_slice(b"W/");
weak.extend_from_slice(bytes);
match HeaderValue::from_bytes(&weak) {
Ok(v) => {
headers.insert(ETAG, v);
}
Err(_) => {
headers.remove(ETAG);
}
}
}
}
}
enum Encoder {
Gzip(flate2::write::GzEncoder<Vec<u8>>),
Br(Box<brotli::CompressorWriter<Vec<u8>>>),
Zstd(zstd::stream::write::Encoder<'static, Vec<u8>>),
}
impl Encoder {
fn new(algo: CompressionAlgorithm) -> std::io::Result<Self> {
match algo {
CompressionAlgorithm::Gzip => Ok(Encoder::Gzip(flate2::write::GzEncoder::new(
Vec::new(),
flate2::Compression::new(GZIP_LEVEL),
))),
CompressionAlgorithm::Br => Ok(Encoder::Br(Box::new(brotli::CompressorWriter::new(
Vec::new(),
BROTLI_BUFFER,
BROTLI_QUALITY,
BROTLI_LGWIN,
)))),
CompressionAlgorithm::Zstd => {
let mut enc = zstd::stream::write::Encoder::new(Vec::new(), ZSTD_LEVEL)?;
enc.set_parameter(zstd::stream::raw::CParameter::WindowLog(ZSTD_WINDOW_LOG))?;
Ok(Encoder::Zstd(enc))
}
}
}
fn write_flush(&mut self, data: &[u8]) -> std::io::Result<Bytes> {
let buf = match self {
Encoder::Gzip(w) => {
w.write_all(data)?;
w.flush()?;
w.get_mut()
}
Encoder::Br(w) => {
w.write_all(data)?;
w.flush()?;
w.get_mut()
}
Encoder::Zstd(w) => {
w.write_all(data)?;
w.flush()?;
w.get_mut()
}
};
Ok(Bytes::from(std::mem::take(buf)))
}
fn finish(self) -> std::io::Result<Bytes> {
let buf = match self {
Encoder::Gzip(w) => w.finish()?,
Encoder::Br(w) => w.into_inner(),
Encoder::Zstd(w) => w.finish()?,
};
Ok(Bytes::from(buf))
}
}
struct CompressBody {
inner: ResponseBody,
encoder: Option<parking_lot::Mutex<Encoder>>,
queue: VecDeque<Frame<Bytes>>,
}
impl hyper::body::Body for CompressBody {
type Data = Bytes;
type Error = BoxError;
fn poll_frame(
self: Pin<&mut Self>,
cx: &mut Context<'_>,
) -> Poll<Option<Result<Frame<Bytes>, BoxError>>> {
let this = self.get_mut();
loop {
if let Some(frame) = this.queue.pop_front() {
return Poll::Ready(Some(Ok(frame)));
}
if this.encoder.is_none() {
return Poll::Ready(None);
}
match std::task::ready!(Pin::new(&mut this.inner).poll_frame(cx)) {
Some(Ok(frame)) => match frame.into_data() {
Ok(data) => {
if data.is_empty() {
continue;
}
let Some(encoder) = this.encoder.as_mut() else {
continue;
};
match encoder.get_mut().write_flush(&data) {
Ok(out) => {
if !out.is_empty() {
this.queue.push_back(Frame::data(out));
}
}
Err(e) => return Poll::Ready(Some(Err(Box::new(e)))),
}
}
Err(other) => {
if let Some(encoder) = this.encoder.take() {
match encoder.into_inner().finish() {
Ok(out) => {
if !out.is_empty() {
this.queue.push_back(Frame::data(out));
}
}
Err(e) => return Poll::Ready(Some(Err(Box::new(e)))),
}
}
this.queue.push_back(other);
}
},
Some(Err(e)) => return Poll::Ready(Some(Err(e))),
None => {
if let Some(encoder) = this.encoder.take() {
match encoder.into_inner().finish() {
Ok(out) => {
if !out.is_empty() {
this.queue.push_back(Frame::data(out));
}
}
Err(e) => return Poll::Ready(Some(Err(Box::new(e)))),
}
}
}
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Read;
fn accept(value: &str) -> HeaderMap {
let mut map = HeaderMap::new();
map.insert(ACCEPT_ENCODING, HeaderValue::from_str(value).unwrap());
map
}
fn all() -> Vec<CompressionAlgorithm> {
vec![
CompressionAlgorithm::Zstd,
CompressionAlgorithm::Br,
CompressionAlgorithm::Gzip,
]
}
#[test]
fn qvalue_grammar_is_enforced() {
for (s, expect) in [
("0", Some(0)),
("0.", Some(0)),
("0.5", Some(500)),
("0.85", Some(850)),
("0.855", Some(855)),
("1", Some(1000)),
("1.0", Some(1000)),
("1.000", Some(1000)),
("1.001", None),
("0.8555", None),
("2", None),
("-1", None),
("abc", None),
("0.x", None),
] {
assert_eq!(parse_qvalue(s), expect, "qvalue {s:?}");
}
}
#[test]
fn negotiate_follows_qvalues_then_server_preference() {
assert_eq!(
negotiate(&accept("gzip;q=1.0, br;q=0.5"), &all()),
Some(CompressionAlgorithm::Gzip)
);
assert_eq!(
negotiate(&accept("gzip, br, zstd"), &all()),
Some(CompressionAlgorithm::Zstd)
);
assert_eq!(negotiate(&accept("gzip;q=0"), &all()), None);
assert_eq!(
negotiate(&accept("*"), &all()),
Some(CompressionAlgorithm::Zstd)
);
assert_eq!(
negotiate(&accept("gzip;q=0.5, *;q=0"), &all()),
Some(CompressionAlgorithm::Gzip)
);
assert_eq!(
negotiate(&accept("zstd;q=0, *;q=0.1"), &all()),
Some(CompressionAlgorithm::Br)
);
assert_eq!(
negotiate(&accept("identity;q=1.0, gzip;q=0.5"), &all()),
None,
"identity preferred → do not transform"
);
assert_eq!(
negotiate(&accept("gzip;q=1.0, identity;q=0.5"), &all()),
Some(CompressionAlgorithm::Gzip),
"gzip preferred → compress"
);
assert_eq!(
negotiate(&accept("identity;q=1.0, gzip;q=1.0"), &all()),
Some(CompressionAlgorithm::Gzip),
"equal q → server preference may compress"
);
assert_eq!(
negotiate(&accept("identity;q=0, gzip"), &all()),
Some(CompressionAlgorithm::Gzip),
"identity excluded → compress"
);
assert_eq!(
negotiate(&accept("gzip, br, zstd"), &all()),
Some(CompressionAlgorithm::Zstd)
);
assert_eq!(negotiate(&HeaderMap::new(), &all()), None);
assert_eq!(
negotiate(&accept("x-gzip"), &all()),
Some(CompressionAlgorithm::Gzip)
);
assert_eq!(negotiate(&accept("gzip;q=banana"), &all()), None);
assert_eq!(negotiate(&accept("gzip;q"), &all()), None);
assert_eq!(
negotiate(&accept("zstd, gzip;q=0.1"), &[CompressionAlgorithm::Gzip]),
Some(CompressionAlgorithm::Gzip)
);
}
fn eligible_headers() -> HeaderMap {
let mut h = HeaderMap::new();
h.insert(CONTENT_TYPE, HeaderValue::from_static("text/html"));
h.insert(CONTENT_LENGTH, HeaderValue::from_static("5000"));
h
}
fn cfg() -> CompressionConfig {
CompressionConfig::new(&plecto_control::RouteCompression {
algorithms: all(),
min_length: 1024,
content_types: vec!["text/html".to_string()],
})
}
#[test]
fn eligibility_is_a_conjunction_of_skips() {
let cfg = cfg();
assert!(response_eligible(StatusCode::OK, &eligible_headers(), &cfg));
for status in [
StatusCode::NO_CONTENT,
StatusCode::PARTIAL_CONTENT,
StatusCode::NOT_MODIFIED,
] {
assert!(
!response_eligible(status, &eligible_headers(), &cfg),
"{status} must not be transformed"
);
}
let mut h = eligible_headers();
h.insert(CONTENT_ENCODING, HeaderValue::from_static("gzip"));
assert!(
!response_eligible(StatusCode::OK, &h, &cfg),
"an already-encoded response is skipped"
);
let mut h = eligible_headers();
h.insert(
CACHE_CONTROL,
HeaderValue::from_static("max-age=60, No-Transform"),
);
assert!(
!response_eligible(StatusCode::OK, &h, &cfg),
"no-transform is a MUST NOT, matched as a token case-insensitively"
);
let mut h = eligible_headers();
h.insert(CONTENT_TYPE, HeaderValue::from_static("image/png"));
assert!(!response_eligible(StatusCode::OK, &h, &cfg));
let mut h = eligible_headers();
h.remove(CONTENT_TYPE);
assert!(
!response_eligible(StatusCode::OK, &h, &cfg),
"no declared type — unknown payloads are left alone"
);
let mut h = eligible_headers();
h.insert(
CONTENT_TYPE,
HeaderValue::from_static("TEXT/HTML; charset=utf-8"),
);
assert!(
response_eligible(StatusCode::OK, &h, &cfg),
"the essence match ignores case and parameters"
);
let mut h = eligible_headers();
h.insert(CONTENT_LENGTH, HeaderValue::from_static("100"));
assert!(!response_eligible(StatusCode::OK, &h, &cfg), "below floor");
let mut h = eligible_headers();
h.insert(CONTENT_LENGTH, HeaderValue::from_static("banana"));
assert!(
!response_eligible(StatusCode::OK, &h, &cfg),
"an unparseable declared length is too weird to touch"
);
let mut h = eligible_headers();
h.remove(CONTENT_LENGTH);
assert!(
response_eligible(StatusCode::OK, &h, &cfg),
"no declared length (chunked / h2 stream) stays eligible"
);
}
#[test]
fn vary_is_added_once_and_respects_existing_declarations() {
let mut h = HeaderMap::new();
add_vary_accept_encoding(&mut h);
add_vary_accept_encoding(&mut h);
assert_eq!(h.get_all(VARY).iter().count(), 1);
let mut h = HeaderMap::new();
h.insert(VARY, HeaderValue::from_static("Accept-Encoding, Origin"));
add_vary_accept_encoding(&mut h);
assert_eq!(h.get_all(VARY).iter().count(), 1, "already declared");
let mut h = HeaderMap::new();
h.insert(VARY, HeaderValue::from_static("*"));
add_vary_accept_encoding(&mut h);
assert_eq!(h.get_all(VARY).iter().count(), 1, "`*` already covers it");
let mut h = HeaderMap::new();
h.insert(VARY, HeaderValue::from_static("origin"));
add_vary_accept_encoding(&mut h);
assert_eq!(h.get_all(VARY).iter().count(), 2, "appended alongside");
}
#[test]
fn mark_compressed_rewrites_the_representation_headers() {
let mut h = eligible_headers();
h.insert(ETAG, HeaderValue::from_static("\"v1\""));
h.insert(ACCEPT_RANGES, HeaderValue::from_static("bytes"));
mark_compressed(&mut h, CompressionAlgorithm::Br);
assert_eq!(h.get(CONTENT_ENCODING).unwrap(), "br");
assert!(h.get(CONTENT_LENGTH).is_none());
assert!(
h.get(ACCEPT_RANGES).is_none(),
"Accept-Ranges named the identity representation — drop it on transform"
);
assert_eq!(h.get(ETAG).unwrap(), "W/\"v1\"");
let mut h = eligible_headers();
h.insert(ETAG, HeaderValue::from_static("W/\"v1\""));
mark_compressed(&mut h, CompressionAlgorithm::Gzip);
assert_eq!(h.get(ETAG).unwrap(), "W/\"v1\"");
}
struct ScriptedBody {
frames: VecDeque<Frame<Bytes>>,
}
impl hyper::body::Body for ScriptedBody {
type Data = Bytes;
type Error = BoxError;
fn poll_frame(
self: Pin<&mut Self>,
_cx: &mut Context<'_>,
) -> Poll<Option<Result<Frame<Bytes>, BoxError>>> {
Poll::Ready(self.get_mut().frames.pop_front().map(Ok))
}
}
fn compress_scripted(
algo: CompressionAlgorithm,
frames: Vec<Frame<Bytes>>,
) -> (Vec<Bytes>, Option<HeaderMap>) {
let inner = http_body_util::BodyExt::boxed(ScriptedBody {
frames: frames.into(),
});
let mut body = CompressBody {
inner,
encoder: Some(parking_lot::Mutex::new(Encoder::new(algo).unwrap())),
queue: VecDeque::new(),
};
let mut data = Vec::new();
let mut trailers = None;
let waker = std::task::Waker::noop();
let mut cx = Context::from_waker(waker);
loop {
match hyper::body::Body::poll_frame(Pin::new(&mut body), &mut cx) {
Poll::Ready(Some(Ok(frame))) => match frame.into_data() {
Ok(d) => data.push(d),
Err(f) => trailers = f.into_trailers().ok(),
},
Poll::Ready(Some(Err(e))) => panic!("body error: {e}"),
Poll::Ready(None) => break,
Poll::Pending => panic!("scripted body never pends"),
}
}
(data, trailers)
}
#[test]
fn every_frame_yields_decodable_bytes_and_the_stream_terminates() {
let frames = vec![
Frame::data(Bytes::from(vec![b'a'; 4096])),
Frame::data(Bytes::from(vec![b'b'; 4096])),
];
let (chunks, trailers) = compress_scripted(CompressionAlgorithm::Gzip, frames);
assert!(
chunks.len() >= 2,
"each input frame must yield its own compressed output (got {})",
chunks.len()
);
assert!(trailers.is_none());
let wire: Vec<u8> = chunks.concat();
let mut out = Vec::new();
flate2::read::GzDecoder::new(wire.as_slice())
.read_to_end(&mut out)
.unwrap();
assert_eq!(out.len(), 8192);
assert!(out[..4096].iter().all(|&b| b == b'a'));
assert!(out[4096..].iter().all(|&b| b == b'b'));
}
#[test]
fn trailers_pass_through_after_the_coded_stream_closes() {
let frames = || {
let mut trailer_map = HeaderMap::new();
trailer_map.insert("grpc-status", HeaderValue::from_static("0"));
vec![
Frame::data(Bytes::from(vec![b'x'; 2048])),
Frame::trailers(trailer_map),
]
};
for algo in all() {
let (chunks, trailers) = compress_scripted(algo, frames());
let trailers = trailers.unwrap_or_else(|| panic!("{algo:?}: trailers dropped"));
assert_eq!(
trailers.get("grpc-status").map(|v| v.as_bytes()),
Some(b"0".as_slice())
);
assert!(!chunks.is_empty(), "{algo:?}: no compressed output");
}
let (chunks, _) = compress_scripted(CompressionAlgorithm::Gzip, frames());
let wire: Vec<u8> = chunks.concat();
let mut out = Vec::new();
flate2::read::GzDecoder::new(wire.as_slice())
.read_to_end(&mut out)
.unwrap();
assert_eq!(out, vec![b'x'; 2048]);
}
#[test]
fn all_three_codecs_roundtrip_via_the_reference_decoders() {
let payload = b"plecto ".repeat(1000);
for algo in all() {
let frames = vec![Frame::data(Bytes::from(payload.clone()))];
let (chunks, _) = compress_scripted(algo, frames);
let wire: Vec<u8> = chunks.concat();
let out = match algo {
CompressionAlgorithm::Gzip => {
let mut out = Vec::new();
flate2::read::GzDecoder::new(wire.as_slice())
.read_to_end(&mut out)
.unwrap();
out
}
CompressionAlgorithm::Br => {
let mut out = Vec::new();
brotli::Decompressor::new(wire.as_slice(), 4096)
.read_to_end(&mut out)
.unwrap();
out
}
CompressionAlgorithm::Zstd => zstd::decode_all(wire.as_slice()).unwrap(),
};
assert_eq!(out, payload, "{algo:?} roundtrip");
assert!(wire.len() < payload.len(), "{algo:?} actually compressed");
}
}
}