static_web_server/directory_listing/
download.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2// This file is part of Static Web Server.
3// See https://static-web-server.net/ for more information
4// Copyright (C) 2019-present Jose Quintana <joseluisq.net>
5
6//! Compress content of a directory into a tarball
7//!
8
9use async_compression::tokio::write::GzipEncoder;
10use async_tar::Builder;
11use clap::ValueEnum;
12use headers::{ContentType, HeaderMapExt};
13use http::{HeaderValue, Method, Response};
14use mime_guess::Mime;
15use std::fmt::Display;
16use std::path::Path;
17use std::path::PathBuf;
18use std::str::FromStr;
19use tokio::fs;
20use tokio::io::AsyncWriteExt;
21use tokio_util::compat::TokioAsyncWriteCompatExt;
22use tokio_util::io::ReaderStream;
23
24use crate::Result;
25use crate::body::Body;
26use crate::exts::http::MethodExt;
27use crate::handler::RequestHandlerOpts;
28
29/// query parameter key to download directory as tar.gz
30pub const DOWNLOAD_PARAM_KEY: &str = "download";
31
32/// Download format for directory
33#[derive(Debug, Serialize, Deserialize, Clone, ValueEnum, Eq, Hash, PartialEq)]
34#[serde(rename_all = "lowercase")]
35pub enum DirDownloadFmt {
36    /// Gunzip-compressed tarball (.tar.gz)
37    Targz,
38}
39
40impl Display for DirDownloadFmt {
41    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
42        std::fmt::Debug::fmt(self, f)
43    }
44}
45
46/// Directory download options.
47pub struct DirDownloadOpts<'a> {
48    /// Request method.
49    pub method: &'a Method,
50    /// Prevent following symlinks for files and directories.
51    pub follow_symlinks: bool,
52    /// Ignore hidden files (dotfiles).
53    pub include_hidden: bool,
54}
55
56/// Initializes directory listing download
57pub fn init(formats: &Vec<DirDownloadFmt>, handler_opts: &mut RequestHandlerOpts) {
58    for fmt in formats {
59        // Use naive implementation since the list is not expected to be long
60        if !handler_opts.dir_listing_download.contains(fmt) {
61            tracing::info!(format = %fmt, "directory listing download format");
62            handler_opts.dir_listing_download.push(fmt.to_owned());
63        }
64    }
65    tracing::info!(
66        enabled = !handler_opts.dir_listing_download.is_empty(),
67        "directory listing download"
68    );
69}
70
71/// It implements `AsyncWrite` backed by a Tokio duplex writer used as the `GzipEncoder` write target.
72pub struct ChannelBuffer {
73    writer: tokio::io::DuplexStream,
74}
75
76impl tokio::io::AsyncWrite for ChannelBuffer {
77    fn poll_write(
78        self: std::pin::Pin<&mut Self>,
79        cx: &mut std::task::Context<'_>,
80        buf: &[u8],
81    ) -> std::task::Poll<Result<usize, std::io::Error>> {
82        std::pin::Pin::new(&mut self.get_mut().writer).poll_write(cx, buf)
83    }
84
85    fn poll_flush(
86        self: std::pin::Pin<&mut Self>,
87        cx: &mut std::task::Context<'_>,
88    ) -> std::task::Poll<Result<(), std::io::Error>> {
89        std::pin::Pin::new(&mut self.get_mut().writer).poll_flush(cx)
90    }
91
92    fn poll_shutdown(
93        self: std::pin::Pin<&mut Self>,
94        cx: &mut std::task::Context<'_>,
95    ) -> std::task::Poll<Result<(), std::io::Error>> {
96        std::pin::Pin::new(&mut self.get_mut().writer).poll_shutdown(cx)
97    }
98}
99
100async fn archive(
101    path: PathBuf,
102    src_path: PathBuf,
103    cb: ChannelBuffer,
104    follow_symlinks: bool,
105    ignore_hidden: bool,
106) -> Result {
107    let gz = GzipEncoder::with_quality(cb, async_compression::Level::Default);
108    let mut a = Builder::new(gz.compat_write());
109    a.follow_symlinks(follow_symlinks);
110
111    // NOTE: Since it is not possible to handle error gracefully, we will
112    // just stop writing when error occurs. It is also not possible to call
113    // sender.abort() as it is protected behind the Builder to ensure
114    // finish() is successfully called.
115
116    // adapted from async_tar::Builder::append_dir_all
117    let mut stack = vec![(src_path.to_path_buf(), true, false)];
118    while let Some((src, is_dir, is_symlink)) = stack.pop() {
119        let dest = path.join(src.strip_prefix(&src_path)?);
120
121        // In case of a symlink pointing to a directory, is_dir is false, but src.is_dir() will return true
122        if is_dir || (is_symlink && follow_symlinks && src.is_dir()) {
123            let mut entries = fs::read_dir(&src).await?;
124            while let Some(entry) = entries.next_entry().await? {
125                // Check and ignore the current hidden file/directory (dotfile) if feature enabled
126                let name = entry.file_name();
127                if ignore_hidden && name.as_encoded_bytes().first().is_some_and(|c| *c == b'.') {
128                    continue;
129                }
130
131                let file_type = entry.file_type().await?;
132                stack.push((entry.path(), file_type.is_dir(), file_type.is_symlink()));
133            }
134            if dest != Path::new("") {
135                a.append_dir(&dest, &src).await?;
136            }
137        } else {
138            // use append_path_with_name to handle symlink
139            a.append_path_with_name(src, &dest).await?;
140        }
141    }
142
143    a.finish().await?;
144    // this is required to emit gzip CRC trailer
145    a.into_inner().await?.into_inner().shutdown().await?;
146
147    Ok(())
148}
149
150/// Reply with archived directory content in compressed tarball format.
151/// The content from `src_path` on server filesystem will be stored to `path`
152/// within the tarball.
153/// An async task will be spawned to asynchronously write compressed data to the
154/// response body.
155pub fn archive_reply<P, Q>(path: P, src_path: Q, opts: DirDownloadOpts<'_>) -> Response<Body>
156where
157    P: AsRef<Path>,
158    Q: AsRef<Path>,
159{
160    let archive_name = path.as_ref().with_extension("tar.gz");
161    let mut resp = Response::new(crate::body::empty());
162
163    resp.headers_mut().typed_insert(ContentType::from(
164        Mime::from_str("application/gzip").unwrap_or(mime_guess::mime::APPLICATION_OCTET_STREAM),
165    ));
166    // SECURITY: Build a safe `Content-Disposition` value that combines an
167    // ASCII-safe quoted-string `filename=...` (for legacy user agents) and
168    // an RFC 5987 `filename*=UTF-8''<percent-encoded>` (for modern UAs).
169    //
170    // The previous implementation interpolated the directory name into the
171    // quoted-string without escaping `"` or `\`, producing a malformed
172    // header for any directory name containing those characters. While
173    // `HeaderValue::from_str` blocks CRLF, malformed Content-Disposition
174    // can still confuse downstream proxies and browsers.
175    let archive_name_str = archive_name.to_string_lossy();
176    let ascii_safe = sanitize_filename_for_quoted_string(&archive_name_str);
177    let percent_encoded = rfc5987_encode_filename(&archive_name_str);
178    let hvals =
179        format!("attachment; filename=\"{ascii_safe}\"; filename*=UTF-8''{percent_encoded}");
180    match HeaderValue::from_str(hvals.as_str()) {
181        Ok(hval) => {
182            resp.headers_mut()
183                .insert(hyper::header::CONTENT_DISPOSITION, hval);
184        }
185        Err(err) => {
186            // not fatal, most browser is able to handle the download since
187            // content-type is set
188            tracing::error!("can't make content disposition from {}: {:?}", hvals, err);
189        }
190    }
191
192    // We skip the body for HEAD requests
193    if opts.method.is_head() {
194        return resp;
195    }
196
197    let (read_half, write_half) = tokio::io::duplex(64 * 1024);
198    let body = crate::body::stream(ReaderStream::new(read_half));
199    tokio::task::spawn(archive(
200        path.as_ref().into(),
201        src_path.as_ref().into(),
202        ChannelBuffer { writer: write_half },
203        opts.follow_symlinks,
204        !opts.include_hidden,
205    ));
206    *resp.body_mut() = body;
207
208    resp
209}
210
211/// Sanitize a filename for use inside an HTTP `Content-Disposition`
212/// `filename="..."` quoted-string. Strips characters that would break the
213/// quoted-string framing (`"`, `\`) or HTTP header parsing (`\r`, `\n`, NUL,
214/// and other ASCII control bytes), and replaces any non-ASCII byte with
215/// `_`. The lossy ASCII filename is paired with an RFC 5987 `filename*=`
216/// variant carrying the full UTF-8 name (see `rfc5987_encode_filename`).
217#[doc(hidden)]
218pub fn sanitize_filename_for_quoted_string(name: &str) -> String {
219    let mut out = String::with_capacity(name.len());
220    for ch in name.chars() {
221        match ch {
222            '"' | '\\' => out.push('_'),
223            c if (c as u32) < 0x20 || c == '\x7f' => out.push('_'),
224            c if c.is_ascii() => out.push(c),
225            _ => out.push('_'),
226        }
227    }
228    if out.is_empty() {
229        out.push_str("download");
230    }
231    out
232}
233
234/// Percent-encode a filename per RFC 5987 (the `attr-char` production
235/// from RFC 8187). Used as the `filename*=UTF-8''<value>` parameter so
236/// non-ASCII filenames survive transit to modern user agents.
237#[doc(hidden)]
238pub fn rfc5987_encode_filename(name: &str) -> String {
239    // RFC 8187 `attr-char` allows: ALPHA / DIGIT and `! # $ & + - . ^ _ ` | ~`
240    // Everything else (including `"`, `\`, space, control bytes, and any
241    // non-ASCII byte) is percent-encoded as `%HH`.
242    fn is_attr_char(b: u8) -> bool {
243        b.is_ascii_alphanumeric()
244            || matches!(
245                b,
246                b'!' | b'#' | b'$' | b'&' | b'+' | b'-' | b'.' | b'^' | b'_' | b'`' | b'|' | b'~'
247            )
248    }
249    let mut out = String::with_capacity(name.len());
250    for &b in name.as_bytes() {
251        if is_attr_char(b) {
252            out.push(b as char);
253        } else {
254            use std::fmt::Write;
255            let _ = write!(out, "%{b:02X}");
256        }
257    }
258    out
259}
260
261#[cfg(test)]
262mod tests {
263    use super::{rfc5987_encode_filename, sanitize_filename_for_quoted_string};
264
265    /// SECURITY: A directory name containing `"` or `\` must NOT break out
266    /// of the `Content-Disposition` quoted-string framing.
267    #[test]
268    fn sanitize_strips_quote_and_backslash() {
269        let out = sanitize_filename_for_quoted_string("evil\".tar.gz");
270        assert!(!out.contains('"'));
271        let out2 = sanitize_filename_for_quoted_string("a\\b.tar.gz");
272        assert!(!out2.contains('\\'));
273    }
274
275    /// SECURITY: Control bytes (CR/LF/NUL) must not survive into a header
276    /// value \u2014 they could be reflected if a downstream proxy mishandles
277    /// `Content-Disposition`.
278    #[test]
279    fn sanitize_strips_control_bytes() {
280        let out = sanitize_filename_for_quoted_string("a\r\nb\tc\x00d");
281        for ch in out.chars() {
282            assert!(
283                ch as u32 >= 0x20 && ch != '\x7f',
284                "control byte leaked: {:?}",
285                ch
286            );
287        }
288    }
289
290    /// Non-ASCII characters are dropped from the quoted-string variant
291    /// (browsers fall back to the `filename*=UTF-8''...` parameter for
292    /// these).
293    #[test]
294    fn sanitize_replaces_non_ascii() {
295        let out = sanitize_filename_for_quoted_string("rep\u{00f6}rt.tar.gz");
296        assert!(out.is_ascii());
297        assert!(out.starts_with("rep_rt") || out.starts_with("rep__rt"));
298    }
299
300    #[test]
301    fn sanitize_never_empty() {
302        assert_eq!(sanitize_filename_for_quoted_string(""), "download");
303    }
304
305    /// RFC 5987 / RFC 8187 attr-char alphabet must round-trip unchanged.
306    #[test]
307    fn rfc5987_preserves_attr_char_alphabet() {
308        let input = "abcXYZ0189!#$&+-.^_`|~";
309        assert_eq!(rfc5987_encode_filename(input), input);
310    }
311
312    /// Everything outside attr-char must be percent-encoded \u2014 in
313    /// particular, `"`, `\`, space, CR, LF, and any non-ASCII byte.
314    #[test]
315    fn rfc5987_encodes_unsafe_bytes() {
316        assert_eq!(rfc5987_encode_filename("a b"), "a%20b");
317        assert_eq!(rfc5987_encode_filename("a\"b"), "a%22b");
318        assert_eq!(rfc5987_encode_filename("a\\b"), "a%5Cb");
319        assert_eq!(rfc5987_encode_filename("a\r\nb"), "a%0D%0Ab");
320        // UTF-8 `\u{00f6}` = 0xC3 0xB6
321        assert_eq!(rfc5987_encode_filename("\u{00f6}"), "%C3%B6");
322    }
323
324    // Property-based regression tests for Content-Disposition helpers.
325    //
326    // These encode the security invariants that protect the
327    // `Content-Disposition` header against quoted-string framing breaks,
328    // header smuggling via control bytes, and ambiguous user-agent
329    // parsing of non-ASCII filenames.
330    use proptest::prelude::*;
331
332    fn is_attr_char(b: u8) -> bool {
333        b.is_ascii_alphanumeric()
334            || matches!(
335                b,
336                b'!' | b'#' | b'$' | b'&' | b'+' | b'-' | b'.' | b'^' | b'_' | b'`' | b'|' | b'~'
337            )
338    }
339
340    proptest! {
341        #![proptest_config(ProptestConfig {
342            cases: 256, ..ProptestConfig::default()
343        })]
344
345        /// `sanitize_filename_for_quoted_string` MUST always yield a
346        /// non-empty ASCII string with no quoted-string-breaking or
347        /// control bytes, for any UTF-8 input.
348        #[test]
349        fn prop_sanitize_filename_invariants(name in "\\PC{0,256}") {
350            let out = sanitize_filename_for_quoted_string(&name);
351            prop_assert!(!out.is_empty(), "output must never be empty");
352            prop_assert!(out.is_ascii(), "output must be pure ASCII");
353            for ch in out.chars() {
354                prop_assert!(
355                    ch != '"' && ch != '\\',
356                    "quoted-string break byte leaked: {:?}",
357                    ch
358                );
359                let code = ch as u32;
360                prop_assert!(
361                    code >= 0x20 && code != 0x7f,
362                    "control byte leaked: {:?}",
363                    ch
364                );
365            }
366        }
367
368        /// Sanitization is idempotent: a single pass already produces a
369        /// fixed point of the transform.
370        #[test]
371        fn prop_sanitize_filename_is_idempotent(name in "\\PC{0,256}") {
372            let once = sanitize_filename_for_quoted_string(&name);
373            let twice = sanitize_filename_for_quoted_string(&once);
374            prop_assert_eq!(once, twice);
375        }
376
377        /// `rfc5987_encode_filename` MUST emit only attr-char bytes or
378        /// well-formed `%HH` percent-escapes, for any UTF-8 input.
379        #[test]
380        fn prop_rfc5987_encode_only_safe_alphabet(name in "\\PC{0,256}") {
381            let out = rfc5987_encode_filename(&name);
382            let bytes = out.as_bytes();
383            let mut i = 0;
384            while i < bytes.len() {
385                let b = bytes[i];
386                if b == b'%' {
387                    // Must be followed by exactly two uppercase hex digits.
388                    prop_assert!(i + 2 < bytes.len(), "truncated percent-escape at {i}");
389                    let h1 = bytes[i + 1];
390                    let h2 = bytes[i + 2];
391                    let is_hex_upper = |c: u8| c.is_ascii_digit() || (b'A'..=b'F').contains(&c);
392                    prop_assert!(
393                        is_hex_upper(h1) && is_hex_upper(h2),
394                        "non-uppercase-hex percent-escape: %{}{}",
395                        h1 as char,
396                        h2 as char
397                    );
398                    i += 3;
399                } else {
400                    prop_assert!(
401                        is_attr_char(b),
402                        "non-attr-char byte leaked: 0x{:02X}",
403                        b
404                    );
405                    i += 1;
406                }
407            }
408        }
409
410        /// Inputs already drawn from the attr-char alphabet must
411        /// round-trip unchanged.
412        #[test]
413        fn prop_rfc5987_attr_char_inputs_roundtrip(s in "[A-Za-z0-9!#\\$&+\\-\\.\\^_`|~]{0,128}") {
414            prop_assert_eq!(rfc5987_encode_filename(&s).clone(), s);
415        }
416    }
417}
static_web_server/directory_listing/download.rs

static_web_server/directory_listing/
download.rs
