static_web_server/
compression.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2// This file is part of Static Web Server.
3// See https://static-web-server.net/ for more information
4// Copyright (C) 2019-present Jose Quintana <joseluisq.net>
5
6//! Auto-compression module to compress responses body.
7//!
8
9// Part of the file is borrowed from <https://github.com/seanmonstar/warp/pull/513>*
10
11#[cfg(any(feature = "compression", feature = "compression-brotli"))]
12use async_compression::tokio::bufread::BrotliEncoder;
13#[cfg(any(feature = "compression", feature = "compression-deflate"))]
14use async_compression::tokio::bufread::DeflateEncoder;
15#[cfg(any(feature = "compression", feature = "compression-gzip"))]
16use async_compression::tokio::bufread::GzipEncoder;
17#[cfg(any(feature = "compression", feature = "compression-zstd"))]
18use async_compression::tokio::bufread::ZstdEncoder;
19
20use headers::{ContentType, HeaderMap, HeaderMapExt, HeaderValue};
21use http_body_util::BodyExt as _;
22use hyper::{
23    Method, Request, Response, StatusCode,
24    header::{CONTENT_ENCODING, CONTENT_LENGTH},
25};
26use mime_guess::Mime;
27use tokio_util::io::{ReaderStream, StreamReader};
28
29use crate::body::Body;
30use crate::error_page;
31use crate::exts::headers::{AcceptEncoding, ContentCoding};
32use crate::exts::http::{MethodExt, append_vary_accept_encoding};
33use crate::exts::mime::MimeExt;
34use crate::handler::RequestHandlerOpts;
35use crate::settings::CompressionLevel;
36use crate::{Error, Result};
37
38/// Minimum response body size in bytes below which dynamic compression is skipped.
39const MIN_COMPRESS_SIZE: usize = 200;
40
41/// List of encodings that can be handled given enabled features.
42const AVAILABLE_ENCODINGS: &[ContentCoding] = &[
43    #[cfg(any(feature = "compression", feature = "compression-deflate"))]
44    ContentCoding::DEFLATE,
45    #[cfg(any(feature = "compression", feature = "compression-gzip"))]
46    ContentCoding::GZIP,
47    #[cfg(any(feature = "compression", feature = "compression-brotli"))]
48    ContentCoding::BROTLI,
49    #[cfg(any(feature = "compression", feature = "compression-zstd"))]
50    ContentCoding::ZSTD,
51];
52
53/// Initializes dynamic compression.
54pub fn init(enabled: bool, level: CompressionLevel, handler_opts: &mut RequestHandlerOpts) {
55    handler_opts.compression = enabled;
56    handler_opts.compression_level = level;
57
58    const FORMATS: &[&str] = &[
59        #[cfg(any(feature = "compression", feature = "compression-deflate"))]
60        "deflate",
61        #[cfg(any(feature = "compression", feature = "compression-gzip"))]
62        "gzip",
63        #[cfg(any(feature = "compression", feature = "compression-brotli"))]
64        "brotli",
65        #[cfg(any(feature = "compression", feature = "compression-zstd"))]
66        "zstd",
67    ];
68    tracing::info!(
69        enabled,
70        formats = %FORMATS.join(","),
71        compression_level = ?level,
72        "auto compression"
73    );
74}
75
76/// Post-processing to dynamically compress the response if necessary.
77pub(crate) fn post_process<T>(
78    opts: &RequestHandlerOpts,
79    req: &Request<T>,
80    mut resp: Response<Body>,
81) -> Result<Response<Body>, Error> {
82    if !opts.compression {
83        return Ok(resp);
84    }
85
86    let is_precompressed = resp.headers().get(CONTENT_ENCODING).is_some();
87    if is_precompressed {
88        return Ok(resp);
89    }
90
91    // Compression content encoding varies so use a `Vary` header
92    append_vary_accept_encoding(&mut resp);
93
94    // Auto compression based on the `Accept-Encoding` header
95    match auto(req.method(), req.headers(), opts.compression_level, resp) {
96        Ok(resp) => Ok(resp),
97        Err(err) => {
98            tracing::error!("error during body compression: {:?}", err);
99            error_page::error_response(
100                req.uri(),
101                req.method(),
102                &StatusCode::INTERNAL_SERVER_ERROR,
103                &opts.page404,
104                &opts.page50x,
105            )
106        }
107    }
108}
109
110/// Create a wrapping handler that compresses the Body of a [`hyper::Response`]
111/// using gzip, `deflate`, `brotli` or `zstd` if is specified in the `Accept-Encoding` header, adding
112/// `content-encoding: <coding>` to the Response's [`HeaderMap`].
113/// It also provides the ability to apply compression for text-based MIME types only.
114pub fn auto(
115    method: &Method,
116    headers: &HeaderMap<HeaderValue>,
117    level: CompressionLevel,
118    resp: Response<Body>,
119) -> Result<Response<Body>> {
120    // Skip compression for HEAD and OPTIONS request methods
121    if method.is_head() || method.is_options() {
122        return Ok(resp);
123    }
124
125    // Compress response based on Accept-Encoding header
126    if let Some(encoding) = get_preferred_encoding(headers) {
127        tracing::trace!(
128            "preferred encoding selected from the accept-encoding header: {:?}",
129            encoding
130        );
131
132        // Skip compression for non-text-based MIME types
133        if let Some(content_type) = resp.headers().typed_get::<ContentType>()
134            && !Mime::from(content_type).is_compressible()
135        {
136            return Ok(resp);
137        }
138
139        // Skip compression for responses below the minimum size threshold.
140        // Tiny payloads gain no benefit and the compression overhead can
141        // make them larger than the original.
142        if let Some(content_length) = resp
143            .headers()
144            .get(CONTENT_LENGTH)
145            .and_then(|v| v.to_str().ok())
146            .and_then(|v| v.parse::<usize>().ok())
147            && content_length < MIN_COMPRESS_SIZE
148        {
149            tracing::trace!(
150                "skipping compression: content-length ({content_length}) below minimum ({MIN_COMPRESS_SIZE})",
151            );
152            return Ok(resp);
153        }
154
155        #[cfg(any(feature = "compression", feature = "compression-gzip"))]
156        if encoding == ContentCoding::GZIP {
157            let (head, body) = resp.into_parts();
158            return Ok(gzip(head, body, level));
159        }
160
161        #[cfg(any(feature = "compression", feature = "compression-deflate"))]
162        if encoding == ContentCoding::DEFLATE {
163            let (head, body) = resp.into_parts();
164            return Ok(deflate(head, body, level));
165        }
166
167        #[cfg(any(feature = "compression", feature = "compression-brotli"))]
168        if encoding == ContentCoding::BROTLI {
169            let (head, body) = resp.into_parts();
170            return Ok(brotli(head, body, level));
171        }
172
173        #[cfg(any(feature = "compression", feature = "compression-zstd"))]
174        if encoding == ContentCoding::ZSTD {
175            let (head, body) = resp.into_parts();
176            return Ok(zstd(head, body, level));
177        }
178
179        tracing::trace!(
180            "no compression feature matched the preferred encoding, probably not enabled or unsupported"
181        );
182    }
183
184    Ok(resp)
185}
186
187/// Create a wrapping handler that compresses the Body of a [`Response`].
188/// using gzip, adding `content-encoding: gzip` to the Response's [`HeaderMap`].
189#[cfg(any(feature = "compression", feature = "compression-gzip"))]
190#[cfg_attr(
191    docsrs,
192    doc(cfg(any(feature = "compression", feature = "compression-gzip")))
193)]
194pub fn gzip(
195    mut head: http::response::Parts,
196    body: Body,
197    level: CompressionLevel,
198) -> Response<Body> {
199    const DEFAULT_COMPRESSION_LEVEL: i32 = 4;
200
201    tracing::trace!("compressing response body on the fly using GZIP");
202
203    let level = level.into_algorithm_level(DEFAULT_COMPRESSION_LEVEL);
204    let body = crate::body::stream(ReaderStream::new(GzipEncoder::with_quality(
205        StreamReader::new(body.into_data_stream()),
206        level,
207    )));
208    let header = create_encoding_header(head.headers.remove(CONTENT_ENCODING), ContentCoding::GZIP);
209    head.headers.remove(CONTENT_LENGTH);
210    head.headers.insert(CONTENT_ENCODING, header);
211    Response::from_parts(head, body)
212}
213
214/// Create a wrapping handler that compresses the Body of a [`Response`].
215/// using deflate, adding `content-encoding: deflate` to the Response's [`HeaderMap`].
216#[cfg(any(feature = "compression", feature = "compression-deflate"))]
217#[cfg_attr(
218    docsrs,
219    doc(cfg(any(feature = "compression", feature = "compression-deflate")))
220)]
221pub fn deflate(
222    mut head: http::response::Parts,
223    body: Body,
224    level: CompressionLevel,
225) -> Response<Body> {
226    const DEFAULT_COMPRESSION_LEVEL: i32 = 4;
227
228    tracing::trace!("compressing response body on the fly using DEFLATE");
229
230    let level = level.into_algorithm_level(DEFAULT_COMPRESSION_LEVEL);
231    let body = crate::body::stream(ReaderStream::new(DeflateEncoder::with_quality(
232        StreamReader::new(body.into_data_stream()),
233        level,
234    )));
235    let header = create_encoding_header(
236        head.headers.remove(CONTENT_ENCODING),
237        ContentCoding::DEFLATE,
238    );
239    head.headers.remove(CONTENT_LENGTH);
240    head.headers.insert(CONTENT_ENCODING, header);
241    Response::from_parts(head, body)
242}
243
244/// Create a wrapping handler that compresses the Body of a [`Response`].
245/// using brotli, adding `content-encoding: br` to the Response's [`HeaderMap`].
246#[cfg(any(feature = "compression", feature = "compression-brotli"))]
247#[cfg_attr(
248    docsrs,
249    doc(cfg(any(feature = "compression", feature = "compression-brotli")))
250)]
251pub fn brotli(
252    mut head: http::response::Parts,
253    body: Body,
254    level: CompressionLevel,
255) -> Response<Body> {
256    const DEFAULT_COMPRESSION_LEVEL: i32 = 4;
257
258    tracing::trace!("compressing response body on the fly using BROTLI");
259
260    let level = level.into_algorithm_level(DEFAULT_COMPRESSION_LEVEL);
261    let body = crate::body::stream(ReaderStream::new(BrotliEncoder::with_quality(
262        StreamReader::new(body.into_data_stream()),
263        level,
264    )));
265    let header =
266        create_encoding_header(head.headers.remove(CONTENT_ENCODING), ContentCoding::BROTLI);
267    head.headers.remove(CONTENT_LENGTH);
268    head.headers.insert(CONTENT_ENCODING, header);
269    Response::from_parts(head, body)
270}
271
272/// Create a wrapping handler that compresses the Body of a [`Response`].
273/// using zstd, adding `content-encoding: zstd` to the Response's [`HeaderMap`].
274#[cfg(any(feature = "compression", feature = "compression-zstd"))]
275#[cfg_attr(
276    docsrs,
277    doc(cfg(any(feature = "compression", feature = "compression-zstd")))
278)]
279pub fn zstd(
280    mut head: http::response::Parts,
281    body: Body,
282    level: CompressionLevel,
283) -> Response<Body> {
284    const DEFAULT_COMPRESSION_LEVEL: i32 = 3;
285
286    tracing::trace!("compressing response body on the fly using ZSTD");
287
288    let level = level.into_algorithm_level(DEFAULT_COMPRESSION_LEVEL);
289    let body = crate::body::stream(ReaderStream::new(ZstdEncoder::with_quality(
290        StreamReader::new(body.into_data_stream()),
291        level,
292    )));
293    let header = create_encoding_header(head.headers.remove(CONTENT_ENCODING), ContentCoding::ZSTD);
294    head.headers.remove(CONTENT_LENGTH);
295    head.headers.insert(CONTENT_ENCODING, header);
296    Response::from_parts(head, body)
297}
298
299/// Given an optional existing encoding header, appends to the existing or creates a new one.
300pub fn create_encoding_header(existing: Option<HeaderValue>, coding: ContentCoding) -> HeaderValue {
301    if let Some(val) = existing
302        && let Ok(str_val) = val.to_str()
303    {
304        return HeaderValue::from_str(&[str_val, ", ", coding.as_str()].concat())
305            .unwrap_or_else(|_| coding.into());
306    }
307    coding.into()
308}
309
310/// Try to get the preferred `content-encoding` via the `accept-encoding` header.
311#[inline(always)]
312pub fn get_preferred_encoding(headers: &HeaderMap<HeaderValue>) -> Option<ContentCoding> {
313    if let Some(ref accept_encoding) = headers.typed_get::<AcceptEncoding>() {
314        tracing::trace!("request with accept-encoding header: {:?}", accept_encoding);
315
316        for encoding in accept_encoding.sorted_encodings() {
317            if AVAILABLE_ENCODINGS.contains(&encoding) {
318                return Some(encoding);
319            }
320        }
321    }
322    None
323}
324
325/// Get the `content-encodings` via the `accept-encoding` header.
326#[inline(always)]
327pub fn get_encodings(headers: &HeaderMap<HeaderValue>) -> Vec<ContentCoding> {
328    if let Some(ref accept_encoding) = headers.typed_get::<AcceptEncoding>() {
329        tracing::trace!("request with accept-encoding header: {:?}", accept_encoding);
330
331        return accept_encoding
332            .sorted_encodings()
333            .filter(|encoding| AVAILABLE_ENCODINGS.contains(encoding))
334            .collect::<Vec<_>>();
335    }
336    vec![]
337}
338
339#[cfg(test)]
340#[cfg(any(feature = "compression", feature = "compression-gzip"))]
341mod tests {
342    use super::*;
343    use crate::body;
344    use crate::settings::CompressionLevel;
345    use http::header::{ACCEPT_ENCODING, CONTENT_TYPE};
346    use hyper::{Method, Response};
347
348    /// Build a `Response<Body>` with a text content-type header,
349    /// a content-length header, and a body of `size` bytes.
350    fn text_response_with_size(size: usize) -> Response<Body> {
351        let body = body::full(vec![b'x'; size]);
352        let mut resp = Response::new(body);
353        resp.headers_mut()
354            .insert(CONTENT_TYPE, "text/html".parse().unwrap());
355        resp.headers_mut()
356            .insert(CONTENT_LENGTH, size.to_string().parse().unwrap());
357        resp
358    }
359
360    /// Build a `Response<Body>` without any `Content-Length` header.
361    fn text_response_without_length() -> Response<Body> {
362        let body = body::full(b"hello world".as_slice());
363        let mut resp = Response::new(body);
364        resp.headers_mut()
365            .insert(CONTENT_TYPE, "text/html".parse().unwrap());
366        resp
367    }
368
369    /// Build a simple GET `HeaderMap` with an Accept-Encoding: gzip header.
370    fn accept_gzip_headers() -> HeaderMap<HeaderValue> {
371        let mut h = HeaderMap::new();
372        h.insert(ACCEPT_ENCODING, "gzip".parse().unwrap());
373        h
374    }
375
376    // Minimum-size threshold tests
377
378    #[test]
379    fn small_response_below_threshold_is_not_compressed() {
380        let resp = text_response_with_size(MIN_COMPRESS_SIZE - 1);
381        let headers = accept_gzip_headers();
382        let result = auto(&Method::GET, &headers, CompressionLevel::Default, resp).unwrap();
383        // no content-encoding should be set
384        assert!(
385            result.headers().get(CONTENT_ENCODING).is_none(),
386            "responses below {MIN_COMPRESS_SIZE} bytes must not be compressed"
387        );
388    }
389
390    #[test]
391    fn response_at_threshold_is_compressed() {
392        let resp = text_response_with_size(MIN_COMPRESS_SIZE);
393        let headers = accept_gzip_headers();
394        let result = auto(&Method::GET, &headers, CompressionLevel::Default, resp).unwrap();
395        assert!(
396            result.headers().get(CONTENT_ENCODING).is_some(),
397            "responses at exactly {MIN_COMPRESS_SIZE} bytes must be compressed"
398        );
399    }
400
401    #[test]
402    fn response_above_threshold_is_compressed() {
403        let resp = text_response_with_size(MIN_COMPRESS_SIZE + 1);
404        let headers = accept_gzip_headers();
405        let result = auto(&Method::GET, &headers, CompressionLevel::Default, resp).unwrap();
406        assert!(
407            result.headers().get(CONTENT_ENCODING).is_some(),
408            "responses above {MIN_COMPRESS_SIZE} bytes must be compressed"
409        );
410    }
411
412    #[test]
413    fn response_without_content_length_is_compressed() {
414        let resp = text_response_without_length();
415        let headers = accept_gzip_headers();
416        let result = auto(&Method::GET, &headers, CompressionLevel::Default, resp).unwrap();
417        assert!(
418            result.headers().get(CONTENT_ENCODING).is_some(),
419            "responses without Content-Length must still be compressed (safe default)"
420        );
421    }
422
423    #[test]
424    fn small_response_head_method_is_not_compressed() {
425        let resp = text_response_with_size(MIN_COMPRESS_SIZE - 1);
426        let headers = accept_gzip_headers();
427        let result = auto(&Method::HEAD, &headers, CompressionLevel::Default, resp).unwrap();
428        assert!(
429            result.headers().get(CONTENT_ENCODING).is_none(),
430            "HEAD requests are never compressed regardless of size"
431        );
432    }
433
434    #[test]
435    fn non_compressible_content_type_is_not_compressed() {
436        let body = body::full(vec![b'x'; MIN_COMPRESS_SIZE + 100]);
437        let mut resp = Response::new(body);
438        resp.headers_mut()
439            .insert(CONTENT_TYPE, "image/png".parse().unwrap());
440        resp.headers_mut().insert(
441            CONTENT_LENGTH,
442            (MIN_COMPRESS_SIZE + 100).to_string().parse().unwrap(),
443        );
444        let headers = accept_gzip_headers();
445        let result = auto(&Method::GET, &headers, CompressionLevel::Default, resp).unwrap();
446        assert!(
447            result.headers().get(CONTENT_ENCODING).is_none(),
448            "non-compressible content-types are never compressed"
449        );
450    }
451}
static_web_server/compression.rs

static_web_server/
compression.rs
