rust_web_server/compression/
dictionary.rs

1//! Dictionary compression (RFC 9842, 18.12.0, #38): a response compressed
2//! against a resource the client already has — usually the previous
3//! version of the same script or stylesheet — so only the difference
4//! crosses the wire.
5//!
6//! The client lists the dictionary it holds in `Available-Dictionary`
7//! (`:<base64 SHA-256>:`) and accepts `dcz` (zstd, the dictionary as a raw
8//! prefix) or `dcb` (brotli, the dictionary as a custom dictionary). The
9//! encoded body starts with a fixed magic and the dictionary's hash, then
10//! the compressed stream.
11//!
12//! ```rust
13//! use rust_web_server::compression::dictionary::{decode, encode, DictionaryEncoding};
14//! use rust_web_server::compression::Level;
15//!
16//! let v1 = b"function greet(name) { return 'Hello, ' + name; }".repeat(20);
17//! let v2 = b"function greet(name) { return 'Hello there, ' + name; }".repeat(20);
18//! # if DictionaryEncoding::Dcz.available() {
19//! let delta = encode(DictionaryEncoding::Dcz, &v2, &v1, Level::Static).unwrap();
20//! assert!(delta.len() < 100);
21//! assert_eq!(decode(DictionaryEncoding::Dcz, &delta, &v1, 1 << 20).unwrap(), v2);
22//! # }
23//! ```
24//!
25//! Static files use this automatically when `compression_dictionaries` is
26//! set; see the Compression docs.
27
28use crate::header::Header;
29use crate::request::Request;
30use crate::response::Response;
31
32use super::Level;
33
34/// The first bytes of a `dcz` body (a zstd skippable frame), before the hash.
35pub const DCZ_MAGIC: [u8; 8] = [0x5e, 0x2a, 0x4d, 0x18, 0x20, 0x00, 0x00, 0x00];
36/// The first bytes of a `dcb` body, before the hash.
37pub const DCB_MAGIC: [u8; 4] = [0xff, 0x44, 0x43, 0x42];
38/// The largest dictionary or body encoded here (1 MiB each). Together they
39/// fit the 8 MiB window browsers support.
40pub const MAX_SIZE: usize = 1024 * 1024;
41
42/// A dictionary's SHA-256, as `Available-Dictionary` names it.
43pub type DictionaryHash = [u8; 32];
44
45/// A dictionary-compressed content coding.
46#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
47pub enum DictionaryEncoding {
48    /// zstd with the dictionary as a raw prefix (needs the `zstd` feature).
49    Dcz,
50    /// Brotli with the dictionary as a custom dictionary (needs `brotli`).
51    Dcb,
52}
53
54impl DictionaryEncoding {
55    /// The `Content-Encoding` / `Accept-Encoding` token.
56    pub fn token(self) -> &'static str {
57        match self {
58            DictionaryEncoding::Dcz => "dcz",
59            DictionaryEncoding::Dcb => "dcb",
60        }
61    }
62
63    /// Whether this build can produce it.
64    pub fn available(self) -> bool {
65        match self {
66            DictionaryEncoding::Dcz => cfg!(feature = "zstd"),
67            DictionaryEncoding::Dcb => cfg!(feature = "brotli"),
68        }
69    }
70
71    fn magic(self) -> &'static [u8] {
72        match self {
73            DictionaryEncoding::Dcz => &DCZ_MAGIC,
74            DictionaryEncoding::Dcb => &DCB_MAGIC,
75        }
76    }
77
78    /// Bytes before the compressed stream: magic and hash.
79    pub fn header_len(self) -> usize {
80        self.magic().len() + 32
81    }
82}
83
84/// On equal weight: zstd (less CPU, and a dictionary of any size), then
85/// brotli.
86pub const PREFERENCE: &[DictionaryEncoding] = &[DictionaryEncoding::Dcz, DictionaryEncoding::Dcb];
87
88/// The hash that names `dictionary`.
89pub fn hash(dictionary: &[u8]) -> DictionaryHash {
90    crate::core::sha256::sha256(dictionary)
91}
92
93/// An `Available-Dictionary` value: a structured-field byte sequence
94/// `:<base64>:` (parameters after it ignored) holding a 32-byte hash.
95pub fn parse_available_dictionary(value: &str) -> Option<DictionaryHash> {
96    let rest = value.trim().strip_prefix(':')?;
97    let end = rest.find(':')?;
98    let (encoded, after) = (&rest[..end], &rest[end + 1..]);
99    if !(after.is_empty() || after.starts_with(';')) || encoded.len() != 44 {
100        return None;
101    }
102    if !encoded.bytes().all(|b| b.is_ascii_alphanumeric() || matches!(b, b'+' | b'/' | b'=')) {
103        return None;
104    }
105    let bytes = crate::core::base64::Base64::decode(encoded.to_string()).ok()?;
106    bytes.try_into().ok()
107}
108
109/// The hash in this request's `Available-Dictionary`, if any.
110pub fn available_dictionary(request: &Request) -> Option<DictionaryHash> {
111    parse_available_dictionary(&request.header("Available-Dictionary")?.value)
112}
113
114/// The dictionary coding to use: the one in [`PREFERENCE`] with the highest
115/// non-zero weight in `Accept-Encoding` (a `*` doesn't count — a client
116/// asks for these by name), compiled in.
117pub fn negotiate(request: &Request) -> Option<DictionaryEncoding> {
118    let header = request.header(Header::_ACCEPT_ENCODING)?;
119    let mut best: Option<(DictionaryEncoding, f32)> = None;
120    for &encoding in PREFERENCE.iter().filter(|e| e.available()) {
121        let listed = header.value.split(',').any(|c| c.split(';').next().unwrap_or("").trim().eq_ignore_ascii_case(encoding.token()));
122        let q = if listed { super::accepted_weight(&header.value, encoding.token()) } else { 0.0 };
123        if q > 0.0 && best.is_none_or(|(_, best_q)| q > best_q) {
124            best = Some((encoding, q));
125        }
126    }
127    best.map(|(encoding, _)| encoding)
128}
129
130/// The smallest power-of-two window, as a log, holding `bytes`.
131fn window_log(bytes: usize) -> u32 {
132    (usize::BITS - bytes.max(1).saturating_sub(1).leading_zeros()).max(10)
133}
134
135/// `body` compressed against `dictionary`, with the magic and hash in
136/// front. `None` when the coding isn't compiled in, either input is over
137/// [`MAX_SIZE`], or the encoder fails.
138pub fn encode(encoding: DictionaryEncoding, body: &[u8], dictionary: &[u8], level: Level) -> Option<Vec<u8>> {
139    if body.len() > MAX_SIZE || dictionary.len() > MAX_SIZE || !encoding.available() {
140        return None;
141    }
142    let mut out = Vec::with_capacity(encoding.header_len() + body.len() / 4 + 64);
143    out.extend_from_slice(encoding.magic());
144    out.extend_from_slice(&hash(dictionary));
145    // The window covers dictionary and body, so the whole dictionary can be
146    // referenced (at most 2 MiB + a little, under the 8 MiB browsers allow).
147    let window = window_log(dictionary.len() + body.len() + 16);
148    match encoding {
149        #[cfg(feature = "zstd")]
150        DictionaryEncoding::Dcz => {
151            let mut compressor = zstd::bulk::Compressor::new(super::ZSTD_LEVEL[level.index()]).ok()?;
152            compressor.set_parameter(zstd::zstd_safe::CParameter::WindowLog(window)).ok()?;
153            compressor.context_mut().ref_prefix(dictionary).ok()?;
154            out.extend_from_slice(&compressor.compress(body).ok()?);
155        }
156        #[cfg(feature = "brotli")]
157        DictionaryEncoding::Dcb => {
158            let params = brotli::enc::BrotliEncoderParams {
159                quality: super::BROTLI_QUALITY[level.index()] as i32,
160                lgwin: window.min(24) as i32,
161                ..Default::default()
162            };
163            let mut nop = |_: &mut brotli::interface::PredictionModeContextMap<brotli::InputReferenceMut>,
164                           _: &mut [brotli::interface::StaticCommand],
165                           _: brotli::InputPair,
166                           _: &mut brotli::enc::StandardAlloc| ();
167            brotli::BrotliCompressCustomIoCustomDict(
168                &mut brotli::enc::IoReaderWrapper(&mut &body[..]),
169                &mut brotli::enc::IoWriterWrapper(&mut out),
170                &mut [0u8; 4096],
171                &mut [0u8; 4096],
172                &params,
173                brotli::enc::StandardAlloc::default(),
174                &mut nop,
175                dictionary,
176                std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "unexpected end of input"),
177            )
178            .ok()?;
179        }
180        // Not compiled in: refused by `available()` above.
181        #[allow(unreachable_patterns)]
182        _ => {
183            let _ = (level, window);
184        }
185    }
186    Some(out)
187}
188
189/// The original of an [`encode`]d body: checks the magic and that the hash
190/// is `dictionary`'s, and stops past `max_len` decoded bytes. `None` on any
191/// mismatch or error.
192#[cfg_attr(not(any(feature = "zstd", feature = "brotli")), allow(unused_variables, unused_mut, unreachable_code))]
193pub fn decode(encoding: DictionaryEncoding, data: &[u8], dictionary: &[u8], max_len: usize) -> Option<Vec<u8>> {
194    let header = encoding.header_len();
195    if data.len() < header || !data.starts_with(encoding.magic()) || data[encoding.magic().len()..header] != hash(dictionary) {
196        return None;
197    }
198    let stream = &data[header..];
199    let mut out = Vec::new();
200    match encoding {
201        #[cfg(feature = "zstd")]
202        DictionaryEncoding::Dcz => {
203            use std::io::Read;
204            let mut decoder = zstd::stream::read::Decoder::with_ref_prefix(stream, dictionary).ok()?;
205            decoder.window_log_max(23).ok()?;
206            decoder.take(max_len as u64 + 1).read_to_end(&mut out).ok()?;
207        }
208        #[cfg(feature = "brotli")]
209        DictionaryEncoding::Dcb => {
210            use brotli::{Allocator, SliceWrapperMut};
211            let mut alloc = brotli::enc::StandardAlloc::default();
212            let mut dict = <brotli::enc::StandardAlloc as Allocator<u8>>::alloc_cell(&mut alloc, dictionary.len());
213            dict.slice_mut().copy_from_slice(dictionary);
214            let mut writer = LimitedWriter { out: &mut out, max: max_len + 1 };
215            brotli::BrotliDecompressCustomIoCustomDict(
216                &mut brotli::enc::IoReaderWrapper(&mut &stream[..]),
217                &mut brotli::enc::IoWriterWrapper(&mut writer),
218                &mut [0u8; 4096],
219                &mut [0u8; 4096],
220                brotli::enc::StandardAlloc::default(),
221                brotli::enc::StandardAlloc::default(),
222                brotli::enc::StandardAlloc::default(),
223                dict,
224                std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "unexpected end of input"),
225            )
226            .ok()?;
227        }
228        #[allow(unreachable_patterns)]
229        _ => {
230            let _ = stream;
231            return None;
232        }
233    }
234    (out.len() <= max_len).then_some(out)
235}
236
237/// A writer that refuses to grow past `max` bytes.
238#[cfg(feature = "brotli")]
239struct LimitedWriter<'a> {
240    out: &'a mut Vec<u8>,
241    max: usize,
242}
243
244#[cfg(feature = "brotli")]
245impl std::io::Write for LimitedWriter<'_> {
246    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
247        if self.out.len() + buf.len() > self.max {
248            return Err(std::io::Error::other("decoded body too large"));
249        }
250        self.out.extend_from_slice(buf);
251        Ok(buf.len())
252    }
253
254    fn flush(&mut self) -> std::io::Result<()> {
255        Ok(())
256    }
257}
258
259/// The headers of a body in `encoding`: `Content-Encoding`, and `Vary` on
260/// `Accept-Encoding` and `Available-Dictionary`.
261pub fn mark_encoded(response: &mut Response, encoding: DictionaryEncoding) {
262    response.headers.push(Header { name: Header::_CONTENT_ENCODING.into(), value: encoding.token().into() });
263    add_vary(response, "Accept-Encoding");
264    add_vary(response, "Available-Dictionary");
265}
266
267/// Append `name` to `Vary` (or add one) unless it's already listed.
268pub(crate) fn add_vary(response: &mut Response, name: &'static str) {
269    if let Some(vary) = response.headers.iter_mut().find(|h| h.name.eq_ignore_ascii_case("vary")) {
270        if !vary.value.split(',').any(|v| v.trim().eq_ignore_ascii_case(name) || v.trim() == "*") {
271            vary.value = format!("{}, {name}", vary.value).into();
272        }
273    } else {
274        response.headers.push(Header { name: "Vary".into(), value: name.into() });
275    }
276}
277
278#[cfg(test)]
279mod tests;
rust_web_server/compression/dictionary.rs

rust_web_server/compression/
dictionary.rs
