rust_web_server/compression/
dictionary.rs

1//! Dictionary compression (RFC 9842, 18.12.0, #38): a response compressed
2//! against a resource the client already has — usually the previous
3//! version of the same script or stylesheet — so only the difference
4//! crosses the wire.
5//!
6//! The client lists the dictionary it holds in `Available-Dictionary`
7//! (`:<base64 SHA-256>:`) and accepts `dcz` (zstd, the dictionary as a raw
8//! prefix) or `dcb` (brotli, the dictionary as a custom dictionary). The
9//! encoded body starts with a fixed magic and the dictionary's hash, then
10//! the compressed stream.
11//!
12//! ```rust
13//! use rust_web_server::compression::dictionary::{decode, encode, DictionaryEncoding};
14//! use rust_web_server::compression::Level;
15//!
16//! let v1 = b"function greet(name) { return 'Hello, ' + name; }".repeat(20);
17//! let v2 = b"function greet(name) { return 'Hello there, ' + name; }".repeat(20);
18//! # if DictionaryEncoding::Dcz.available() {
19//! let delta = encode(DictionaryEncoding::Dcz, &v2, &v1, Level::Static).unwrap();
20//! assert!(delta.len() < 100);
21//! assert_eq!(decode(DictionaryEncoding::Dcz, &delta, &v1, 1 << 20).unwrap(), v2);
22//! # }
23//! ```
24//!
25//! Static files use this automatically when `compression_dictionaries` is
26//! set; see the Compression docs.
27
28use crate::header::Header;
29use crate::request::Request;
30use crate::response::Response;
31
32use super::Level;
33
34/// The first bytes of a `dcz` body (a zstd skippable frame), before the hash.
35pub const DCZ_MAGIC: [u8; 8] = [0x5e, 0x2a, 0x4d, 0x18, 0x20, 0x00, 0x00, 0x00];
36/// The first bytes of a `dcb` body, before the hash.
37pub const DCB_MAGIC: [u8; 4] = [0xff, 0x44, 0x43, 0x42];
38/// The largest dictionary or body encoded here (1 MiB each). Together they
39/// fit the 8 MiB window browsers support.
40pub const MAX_SIZE: usize = 1024 * 1024;
41
42/// A dictionary's SHA-256, as `Available-Dictionary` names it.
43pub type DictionaryHash = [u8; 32];
44
45/// A dictionary-compressed content coding.
46#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
47pub enum DictionaryEncoding {
48    /// zstd with the dictionary as a raw prefix (needs the `zstd` feature).
49    Dcz,
50    /// Brotli with the dictionary as a custom dictionary (needs `brotli`).
51    Dcb,
52}
53
54impl DictionaryEncoding {
55    /// The `Content-Encoding` / `Accept-Encoding` token.
56    pub fn token(self) -> &'static str {
57        match self {
58            DictionaryEncoding::Dcz => "dcz",
59            DictionaryEncoding::Dcb => "dcb",
60        }
61    }
62
63    /// Whether this build can produce it.
64    pub fn available(self) -> bool {
65        match self {
66            DictionaryEncoding::Dcz => cfg!(feature = "zstd"),
67            DictionaryEncoding::Dcb => cfg!(feature = "brotli"),
68        }
69    }
70
71    fn magic(self) -> &'static [u8] {
72        match self {
73            DictionaryEncoding::Dcz => &DCZ_MAGIC,
74            DictionaryEncoding::Dcb => &DCB_MAGIC,
75        }
76    }
77
78    /// Bytes before the compressed stream: magic and hash.
79    pub fn header_len(self) -> usize {
80        self.magic().len() + 32
81    }
82}
83
84/// On equal weight: zstd (less CPU, and a dictionary of any size), then
85/// brotli.
86pub const PREFERENCE: &[DictionaryEncoding] = &[DictionaryEncoding::Dcz, DictionaryEncoding::Dcb];
87
88/// The hash that names `dictionary`.
89pub fn hash(dictionary: &[u8]) -> DictionaryHash {
90    crate::core::sha256::sha256(dictionary)
91}
92
93/// An `Available-Dictionary` value: a structured-field byte sequence
94/// `:<base64>:` (parameters after it ignored) holding a 32-byte hash.
95pub fn parse_available_dictionary(value: &str) -> Option<DictionaryHash> {
96    let rest = value.trim().strip_prefix(':')?;
97    let end = rest.find(':')?;
98    let (encoded, after) = (&rest[..end], &rest[end + 1..]);
99    if !(after.is_empty() || after.starts_with(';')) || encoded.len() != 44 {
100        return None;
101    }
102    if !encoded.bytes().all(|b| b.is_ascii_alphanumeric() || matches!(b, b'+' | b'/' | b'=')) {
103        return None;
104    }
105    let bytes = crate::core::base64::Base64::decode(encoded.to_string()).ok()?;
106    bytes.try_into().ok()
107}
108
109/// The hash in this request's `Available-Dictionary`, if any.
110pub fn available_dictionary(request: &Request) -> Option<DictionaryHash> {
111    parse_available_dictionary(&request.header("Available-Dictionary")?.value)
112}
113
114/// The dictionary coding to use: the one in [`PREFERENCE`] with the highest
115/// non-zero weight in `Accept-Encoding` (a `*` doesn't count — a client
116/// asks for these by name), compiled in.
117pub fn negotiate(request: &Request) -> Option<DictionaryEncoding> {
118    let header = request.header(Header::_ACCEPT_ENCODING)?;
119    let mut best: Option<(DictionaryEncoding, f32)> = None;
120    for &encoding in PREFERENCE.iter().filter(|e| e.available()) {
121        let listed = header.value.split(',').any(|c| c.split(';').next().unwrap_or("").trim().eq_ignore_ascii_case(encoding.token()));
122        let q = if listed { super::accepted_weight(&header.value, encoding.token()) } else { 0.0 };
123        if q > 0.0 && best.is_none_or(|(_, best_q)| q > best_q) {
124            best = Some((encoding, q));
125        }
126    }
127    best.map(|(encoding, _)| encoding)
128}
129
130/// The smallest power-of-two window, as a log, holding `bytes`.
131fn window_log(bytes: usize) -> u32 {
132    (usize::BITS - bytes.max(1).saturating_sub(1).leading_zeros()).max(10)
133}
134
135/// `body` compressed against `dictionary`, with the magic and hash in
136/// front. `None` when the coding isn't compiled in, either input is over
137/// [`MAX_SIZE`], or the encoder fails.
138pub fn encode(encoding: DictionaryEncoding, body: &[u8], dictionary: &[u8], level: Level) -> Option<Vec<u8>> {
139    if body.len() > MAX_SIZE || dictionary.len() > MAX_SIZE || !encoding.available() {
140        return None;
141    }
142    let mut out = Vec::with_capacity(encoding.header_len() + body.len() / 4 + 64);
143    out.extend_from_slice(encoding.magic());
144    out.extend_from_slice(&hash(dictionary));
145    // The window covers dictionary and body, so the whole dictionary can be
146    // referenced (at most 2 MiB + a little, under the 8 MiB browsers allow).
147    let window = window_log(dictionary.len() + body.len() + 16);
148    match encoding {
149        #[cfg(feature = "zstd")]
150        DictionaryEncoding::Dcz => {
151            let mut compressor = zstd::bulk::Compressor::new(super::ZSTD_LEVEL[level.index()]).ok()?;
152            compressor.set_parameter(zstd::zstd_safe::CParameter::WindowLog(window)).ok()?;
153            compressor.context_mut().ref_prefix(dictionary).ok()?;
154            out.extend_from_slice(&compressor.compress(body).ok()?);
155        }
156        #[cfg(feature = "brotli")]
157        DictionaryEncoding::Dcb => {
158            let params = brotli::enc::BrotliEncoderParams {
159                quality: super::BROTLI_QUALITY[level.index()] as i32,
160                lgwin: window.min(24) as i32,
161                ..Default::default()
162            };
163            let mut nop = |_: &mut brotli::interface::PredictionModeContextMap<brotli::InputReferenceMut>,
164                           _: &mut [brotli::interface::StaticCommand],
165                           _: brotli::InputPair,
166                           _: &mut brotli::enc::StandardAlloc| ();
167            brotli::BrotliCompressCustomIoCustomDict(
168                &mut brotli::enc::IoReaderWrapper(&mut &body[..]),
169                &mut brotli::enc::IoWriterWrapper(&mut out),
170                &mut [0u8; 4096],
171                &mut [0u8; 4096],
172                &params,
173                brotli::enc::StandardAlloc::default(),
174                &mut nop,
175                dictionary,
176                std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "unexpected end of input"),
177            )
178            .ok()?;
179        }
180        #[allow(unreachable_patterns)]
181        _ => {
182            let _ = (level, window);
183            return None;
184        }
185    }
186    Some(out)
187}
188
189/// The original of an [`encode`]d body: checks the magic and that the hash
190/// is `dictionary`'s, and stops past `max_len` decoded bytes. `None` on any
191/// mismatch or error.
192pub fn decode(encoding: DictionaryEncoding, data: &[u8], dictionary: &[u8], max_len: usize) -> Option<Vec<u8>> {
193    use std::io::Read;
194    let header = encoding.header_len();
195    if data.len() < header || !data.starts_with(encoding.magic()) || data[encoding.magic().len()..header] != hash(dictionary) {
196        return None;
197    }
198    let stream = &data[header..];
199    let mut out = Vec::new();
200    match encoding {
201        #[cfg(feature = "zstd")]
202        DictionaryEncoding::Dcz => {
203            let mut decoder = zstd::stream::read::Decoder::with_ref_prefix(stream, dictionary).ok()?;
204            decoder.window_log_max(23).ok()?;
205            decoder.take(max_len as u64 + 1).read_to_end(&mut out).ok()?;
206        }
207        #[cfg(feature = "brotli")]
208        DictionaryEncoding::Dcb => {
209            use brotli::{Allocator, SliceWrapperMut};
210            let mut alloc = brotli::enc::StandardAlloc::default();
211            let mut dict = <brotli::enc::StandardAlloc as Allocator<u8>>::alloc_cell(&mut alloc, dictionary.len());
212            dict.slice_mut().copy_from_slice(dictionary);
213            let mut writer = LimitedWriter { out: &mut out, max: max_len + 1 };
214            brotli::BrotliDecompressCustomIoCustomDict(
215                &mut brotli::enc::IoReaderWrapper(&mut &stream[..]),
216                &mut brotli::enc::IoWriterWrapper(&mut writer),
217                &mut [0u8; 4096],
218                &mut [0u8; 4096],
219                brotli::enc::StandardAlloc::default(),
220                brotli::enc::StandardAlloc::default(),
221                brotli::enc::StandardAlloc::default(),
222                dict,
223                std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "unexpected end of input"),
224            )
225            .ok()?;
226        }
227        #[allow(unreachable_patterns)]
228        _ => return None,
229    }
230    (out.len() <= max_len).then_some(out)
231}
232
233/// A writer that refuses to grow past `max` bytes.
234#[cfg(feature = "brotli")]
235struct LimitedWriter<'a> {
236    out: &'a mut Vec<u8>,
237    max: usize,
238}
239
240#[cfg(feature = "brotli")]
241impl std::io::Write for LimitedWriter<'_> {
242    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
243        if self.out.len() + buf.len() > self.max {
244            return Err(std::io::Error::other("decoded body too large"));
245        }
246        self.out.extend_from_slice(buf);
247        Ok(buf.len())
248    }
249
250    fn flush(&mut self) -> std::io::Result<()> {
251        Ok(())
252    }
253}
254
255/// The headers of a body in `encoding`: `Content-Encoding`, and `Vary` on
256/// `Accept-Encoding` and `Available-Dictionary`.
257pub fn mark_encoded(response: &mut Response, encoding: DictionaryEncoding) {
258    response.headers.push(Header { name: Header::_CONTENT_ENCODING.into(), value: encoding.token().into() });
259    add_vary(response, "Accept-Encoding");
260    add_vary(response, "Available-Dictionary");
261}
262
263/// Append `name` to `Vary` (or add one) unless it's already listed.
264pub(crate) fn add_vary(response: &mut Response, name: &'static str) {
265    if let Some(vary) = response.headers.iter_mut().find(|h| h.name.eq_ignore_ascii_case("vary")) {
266        if !vary.value.split(',').any(|v| v.trim().eq_ignore_ascii_case(name) || v.trim() == "*") {
267            vary.value = format!("{}, {name}", vary.value).into();
268        }
269    } else {
270        response.headers.push(Header { name: "Vary".into(), value: name.into() });
271    }
272}
273
274#[cfg(test)]
275mod tests;
rust_web_server/compression/dictionary.rs

rust_web_server/compression/
dictionary.rs
