client_util/request/
multipart.rs

1/*
2
3I basically copied the code from the reqwest crate and pasted it here, and modify it to work with the hyper crate.
4The source file is here: https://github.com/seanmonstar/reqwest/blob/master/src/async_impl/multipart.rs
5
6*/
7
8//! multipart/form-data
9use std::borrow::Cow;
10use std::fmt;
11use std::pin::Pin;
12
13use bytes::Bytes;
14use http_body::Frame;
15use mime::Mime;
16use percent_encoding::{self, AsciiSet, NON_ALPHANUMERIC};
17
18use crate::body::{boxed_empty, boxed_full, boxed_stream, Body};
19use crate::request::BuildMultipartError;
20use futures_core::Stream;
21use futures_util::{future, stream, StreamExt};
22use http::HeaderMap;
23use http_body_util::BodyStream;
24
25/// An async multipart/form-data request.
26pub struct Form {
27    inner: FormParts<Part>,
28}
29
30/// A field in a multipart form.
31pub struct Part {
32    meta: PartMetadata,
33    value: Body,
34    body_length: Option<u64>,
35}
36
37pub(crate) struct FormParts<P> {
38    pub(crate) boundary: String,
39    pub(crate) computed_headers: Vec<Vec<u8>>,
40    pub(crate) fields: Vec<(Cow<'static, str>, P)>,
41    pub(crate) percent_encoding: PercentEncoding,
42}
43
44pub(crate) struct PartMetadata {
45    mime: Option<Mime>,
46    file_name: Option<Cow<'static, str>>,
47    pub(crate) headers: HeaderMap,
48}
49
50pub(crate) trait PartProps {
51    fn value_len(&self) -> Option<u64>;
52    fn metadata(&self) -> &PartMetadata;
53}
54
55// ===== impl Form =====
56
57impl Default for Form {
58    fn default() -> Self {
59        Self::new()
60    }
61}
62
63impl Form {
64    /// Creates a new async Form without any content.
65    pub fn new() -> Form {
66        Form {
67            inner: FormParts::new(),
68        }
69    }
70
71    /// Get the boundary that this form will use.
72    #[inline]
73    pub fn boundary(&self) -> &str {
74        self.inner.boundary()
75    }
76
77    /// Add a data field with supplied name and value.
78    ///
79    /// # Examples
80    ///
81    /// ```
82    /// # use client_util::prelude::*;
83    /// let form = Form::new()
84    ///     .text("username", "seanmonstar")
85    ///     .text("password", "secret");
86    /// ```
87    pub fn text<T, U>(self, name: T, value: U) -> Form
88    where
89        T: Into<Cow<'static, str>>,
90        U: Into<Cow<'static, str>>,
91    {
92        self.part(name, Part::text(value))
93    }
94
95    /// Adds a customized Part.
96    pub fn part<T>(self, name: T, part: Part) -> Form
97    where
98        T: Into<Cow<'static, str>>,
99    {
100        self.with_inner(move |inner| inner.part(name, part))
101    }
102
103    /// Configure this `Form` to percent-encode using the `path-segment` rules.
104    pub fn percent_encode_path_segment(self) -> Form {
105        self.with_inner(|inner| inner.percent_encode_path_segment())
106    }
107
108    /// Configure this `Form` to percent-encode using the `attr-char` rules.
109    pub fn percent_encode_attr_chars(self) -> Form {
110        self.with_inner(|inner| inner.percent_encode_attr_chars())
111    }
112
113    /// Configure this `Form` to skip percent-encoding
114    pub fn percent_encode_noop(self) -> Form {
115        self.with_inner(|inner| inner.percent_encode_noop())
116    }
117
118    /// Consume this instance and transform into an instance of Body for use in a request.
119    pub(crate) fn stream(mut self) -> Body {
120        if self.inner.fields.is_empty() {
121            return boxed_empty();
122        }
123
124        // create initial part to init reduce chain
125        let (name, part) = self.inner.fields.remove(0);
126        let start = Box::pin(self.part_stream(name, part))
127            as Pin<
128                Box<dyn Stream<Item = Result<Frame<Bytes>, crate::error::BoxError>> + Send + Sync>,
129            >;
130
131        let fields = self.inner.take_fields();
132        // for each field, chain an additional stream
133        let stream = fields.into_iter().fold(start, |memo, (name, part)| {
134            let part_stream = self.part_stream(name, part);
135            Box::pin(memo.chain(part_stream))
136                as Pin<
137                    Box<
138                        dyn Stream<Item = Result<Frame<Bytes>, crate::error::BoxError>>
139                            + Send
140                            + Sync,
141                    >,
142                >
143        });
144        // append special ending boundary
145        let last = stream::once(future::ready(Ok(Frame::data(
146            format!("--{}--\r\n", self.boundary()).into(),
147        ))));
148        boxed_stream(stream.chain(last))
149    }
150
151    /// Generate a hyper::Body stream for a single Part instance of a Form request.
152    pub(crate) fn part_stream<T>(
153        &mut self,
154        name: T,
155        part: Part,
156    ) -> impl Stream<Item = Result<Frame<Bytes>, crate::error::BoxError>> + Send + Sync
157    where
158        T: Into<Cow<'static, str>>,
159    {
160        // start with boundary
161        let boundary = stream::once(future::ready(Ok(Frame::data(Bytes::from(format!(
162            "--{}\r\n",
163            self.boundary()
164        ))))));
165        // append headers
166        let header = stream::once(future::ready(Ok({
167            let mut h = self
168                .inner
169                .percent_encoding
170                .encode_headers(&name.into(), &part.meta);
171            h.extend_from_slice(b"\r\n\r\n");
172            Frame::data(Bytes::from(h))
173        })));
174        // then append form data followed by terminating CRLF
175        boundary
176            .chain(header)
177            .chain(BodyStream::new(part.value))
178            .chain(stream::once(future::ready(Ok(Frame::data(
179                Bytes::from_static(b"\r\n"),
180            )))))
181    }
182
183    pub(crate) fn compute_length(&mut self) -> Option<u64> {
184        self.inner.compute_length()
185    }
186
187    fn with_inner<F>(self, func: F) -> Self
188    where
189        F: FnOnce(FormParts<Part>) -> FormParts<Part>,
190    {
191        Form {
192            inner: func(self.inner),
193        }
194    }
195}
196
197impl fmt::Debug for Form {
198    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
199        self.inner.fmt_fields("Form", f)
200    }
201}
202
203// ===== impl Part =====
204
205impl Part {
206    /// Makes a text parameter.
207    pub fn text<T>(value: T) -> Part
208    where
209        T: Into<Cow<'static, str>>,
210    {
211        let cow = value.into();
212        let len = cow.len() as u64;
213        let body = match cow {
214            Cow::Borrowed(slice) => boxed_full(slice),
215            Cow::Owned(string) => boxed_full(string),
216        };
217        Part::new(body, Some(len))
218    }
219
220    /// Makes a new parameter from arbitrary bytes.
221    pub fn bytes<T>(value: T) -> Part
222    where
223        T: Into<Cow<'static, [u8]>>,
224    {
225        let cow = value.into();
226        let length = cow.len() as u64;
227        let body = match cow {
228            Cow::Borrowed(slice) => boxed_full(slice),
229            Cow::Owned(vec) => boxed_full(vec),
230        };
231        Part::new(body, Some(length))
232    }
233
234    /// Makes a new parameter from an arbitrary stream.
235    pub fn body<T: Into<Body>>(value: T) -> Part {
236        Part::new(value.into(), None)
237    }
238
239    /// Makes a new parameter from an arbitrary stream with a known length. This is particularly
240    /// useful when adding something like file contents as a stream, where you can know the content
241    /// length beforehand.
242    pub fn body_with_length<T: Into<Body>>(value: T, length: u64) -> Part {
243        Part::new(value.into(), Some(length))
244    }
245
246    fn new(value: Body, body_length: Option<u64>) -> Part {
247        Part {
248            meta: PartMetadata::new(),
249            value,
250            body_length,
251        }
252    }
253
254    /// Tries to set the mime of this part.
255    pub fn mime_str(self, mime: &str) -> Result<Part, BuildMultipartError> {
256        Ok(self.mime(mime.parse()?))
257    }
258
259    // Re-export when mime 0.4 is available, with split MediaType/MediaRange.
260    fn mime(self, mime: Mime) -> Part {
261        self.with_inner(move |inner| inner.mime(mime))
262    }
263
264    /// Sets the filename, builder style.
265    pub fn file_name<T>(self, filename: T) -> Part
266    where
267        T: Into<Cow<'static, str>>,
268    {
269        self.with_inner(move |inner| inner.file_name(filename))
270    }
271
272    /// Sets custom headers for the part.
273    pub fn headers(self, headers: HeaderMap) -> Part {
274        self.with_inner(move |inner| inner.headers(headers))
275    }
276
277    fn with_inner<F>(self, func: F) -> Self
278    where
279        F: FnOnce(PartMetadata) -> PartMetadata,
280    {
281        Part {
282            meta: func(self.meta),
283            ..self
284        }
285    }
286}
287
288impl fmt::Debug for Part {
289    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
290        let mut dbg = f.debug_struct("Part");
291        dbg.field("value", &self.value);
292        self.meta.fmt_fields(&mut dbg);
293        dbg.finish()
294    }
295}
296
297impl PartProps for Part {
298    fn value_len(&self) -> Option<u64> {
299        if self.body_length.is_some() {
300            self.body_length
301        } else {
302            // self.value.content_length()
303            None
304        }
305    }
306
307    fn metadata(&self) -> &PartMetadata {
308        &self.meta
309    }
310}
311
312// ===== impl FormParts =====
313
314impl<P: PartProps> FormParts<P> {
315    pub(crate) fn new() -> Self {
316        FormParts {
317            boundary: gen_boundary(),
318            computed_headers: Vec::new(),
319            fields: Vec::new(),
320            percent_encoding: PercentEncoding::PathSegment,
321        }
322    }
323
324    pub(crate) fn boundary(&self) -> &str {
325        &self.boundary
326    }
327
328    /// Adds a customized Part.
329    pub(crate) fn part<T>(mut self, name: T, part: P) -> Self
330    where
331        T: Into<Cow<'static, str>>,
332    {
333        self.fields.push((name.into(), part));
334        self
335    }
336
337    /// Configure this `Form` to percent-encode using the `path-segment` rules.
338    pub(crate) fn percent_encode_path_segment(mut self) -> Self {
339        self.percent_encoding = PercentEncoding::PathSegment;
340        self
341    }
342
343    /// Configure this `Form` to percent-encode using the `attr-char` rules.
344    pub(crate) fn percent_encode_attr_chars(mut self) -> Self {
345        self.percent_encoding = PercentEncoding::AttrChar;
346        self
347    }
348
349    /// Configure this `Form` to skip percent-encoding
350    pub(crate) fn percent_encode_noop(mut self) -> Self {
351        self.percent_encoding = PercentEncoding::NoOp;
352        self
353    }
354
355    // If predictable, computes the length the request will have
356    // The length should be preditable if only String and file fields have been added,
357    // but not if a generic reader has been added;
358    pub(crate) fn compute_length(&mut self) -> Option<u64> {
359        let mut length = 0u64;
360        for (name, field) in self.fields.iter() {
361            match field.value_len() {
362                Some(value_length) => {
363                    // We are constructing the header just to get its length. To not have to
364                    // construct it again when the request is sent we cache these headers.
365                    let header = self.percent_encoding.encode_headers(name, field.metadata());
366                    let header_length = header.len();
367                    self.computed_headers.push(header);
368                    // The additions mimic the format string out of which the field is constructed
369                    // in Reader. Not the cleanest solution because if that format string is
370                    // ever changed then this formula needs to be changed too which is not an
371                    // obvious dependency in the code.
372                    length += 2
373                        + self.boundary().len() as u64
374                        + 2
375                        + header_length as u64
376                        + 4
377                        + value_length
378                        + 2
379                }
380                _ => return None,
381            }
382        }
383        // If there is a at least one field there is a special boundary for the very last field.
384        if !self.fields.is_empty() {
385            length += 2 + self.boundary().len() as u64 + 4
386        }
387        Some(length)
388    }
389
390    /// Take the fields vector of this instance, replacing with an empty vector.
391    fn take_fields(&mut self) -> Vec<(Cow<'static, str>, P)> {
392        std::mem::take(&mut self.fields)
393    }
394}
395
396impl<P: fmt::Debug> FormParts<P> {
397    pub(crate) fn fmt_fields(&self, ty_name: &'static str, f: &mut fmt::Formatter) -> fmt::Result {
398        f.debug_struct(ty_name)
399            .field("boundary", &self.boundary)
400            .field("parts", &self.fields)
401            .finish()
402    }
403}
404
405// ===== impl PartMetadata =====
406
407impl PartMetadata {
408    pub(crate) fn new() -> Self {
409        PartMetadata {
410            mime: None,
411            file_name: None,
412            headers: HeaderMap::default(),
413        }
414    }
415
416    pub(crate) fn mime(mut self, mime: Mime) -> Self {
417        self.mime = Some(mime);
418        self
419    }
420
421    pub(crate) fn file_name<T>(mut self, filename: T) -> Self
422    where
423        T: Into<Cow<'static, str>>,
424    {
425        self.file_name = Some(filename.into());
426        self
427    }
428
429    pub(crate) fn headers<T>(mut self, headers: T) -> Self
430    where
431        T: Into<HeaderMap>,
432    {
433        self.headers = headers.into();
434        self
435    }
436}
437
438impl PartMetadata {
439    pub(crate) fn fmt_fields<'f, 'fa, 'fb>(
440        &self,
441        debug_struct: &'f mut fmt::DebugStruct<'fa, 'fb>,
442    ) -> &'f mut fmt::DebugStruct<'fa, 'fb> {
443        debug_struct
444            .field("mime", &self.mime)
445            .field("file_name", &self.file_name)
446            .field("headers", &self.headers)
447    }
448}
449
450// https://url.spec.whatwg.org/#fragment-percent-encode-set
451const FRAGMENT_ENCODE_SET: &AsciiSet = &percent_encoding::CONTROLS
452    .add(b' ')
453    .add(b'"')
454    .add(b'<')
455    .add(b'>')
456    .add(b'`');
457
458// https://url.spec.whatwg.org/#path-percent-encode-set
459const PATH_ENCODE_SET: &AsciiSet = &FRAGMENT_ENCODE_SET.add(b'#').add(b'?').add(b'{').add(b'}');
460
461const PATH_SEGMENT_ENCODE_SET: &AsciiSet = &PATH_ENCODE_SET.add(b'/').add(b'%');
462
463// https://tools.ietf.org/html/rfc8187#section-3.2.1
464const ATTR_CHAR_ENCODE_SET: &AsciiSet = &NON_ALPHANUMERIC
465    .remove(b'!')
466    .remove(b'#')
467    .remove(b'$')
468    .remove(b'&')
469    .remove(b'+')
470    .remove(b'-')
471    .remove(b'.')
472    .remove(b'^')
473    .remove(b'_')
474    .remove(b'`')
475    .remove(b'|')
476    .remove(b'~');
477
478pub(crate) enum PercentEncoding {
479    PathSegment,
480    AttrChar,
481    NoOp,
482}
483
484impl PercentEncoding {
485    pub(crate) fn encode_headers(&self, name: &str, field: &PartMetadata) -> Vec<u8> {
486        let mut buf = Vec::new();
487        buf.extend_from_slice(b"Content-Disposition: form-data; ");
488
489        match self.percent_encode(name) {
490            Cow::Borrowed(value) => {
491                // nothing has been percent encoded
492                buf.extend_from_slice(b"name=\"");
493                buf.extend_from_slice(value.as_bytes());
494                buf.extend_from_slice(b"\"");
495            }
496            Cow::Owned(value) => {
497                // something has been percent encoded
498                buf.extend_from_slice(b"name*=utf-8''");
499                buf.extend_from_slice(value.as_bytes());
500            }
501        }
502
503        // According to RFC7578 Section 4.2, `filename*=` syntax is invalid.
504        // See https://github.com/seanmonstar/reqwest/issues/419.
505        if let Some(filename) = &field.file_name {
506            buf.extend_from_slice(b"; filename=\"");
507            let legal_filename = filename
508                .replace('\\', "\\\\")
509                .replace('"', "\\\"")
510                .replace('\r', "\\\r")
511                .replace('\n', "\\\n");
512            buf.extend_from_slice(legal_filename.as_bytes());
513            buf.extend_from_slice(b"\"");
514        }
515
516        if let Some(mime) = &field.mime {
517            buf.extend_from_slice(b"\r\nContent-Type: ");
518            buf.extend_from_slice(mime.as_ref().as_bytes());
519        }
520
521        for (k, v) in field.headers.iter() {
522            buf.extend_from_slice(b"\r\n");
523            buf.extend_from_slice(k.as_str().as_bytes());
524            buf.extend_from_slice(b": ");
525            buf.extend_from_slice(v.as_bytes());
526        }
527        buf
528    }
529
530    fn percent_encode<'a>(&self, value: &'a str) -> Cow<'a, str> {
531        use percent_encoding::utf8_percent_encode as percent_encode;
532
533        match self {
534            Self::PathSegment => percent_encode(value, PATH_SEGMENT_ENCODE_SET).into(),
535            Self::AttrChar => percent_encode(value, ATTR_CHAR_ENCODE_SET).into(),
536            Self::NoOp => value.into(),
537        }
538    }
539}
540
541fn gen_boundary() -> String {
542    // use crate::util::fast_random as random;
543
544    let a = crate::util::fast_random();
545    let b = crate::util::fast_random();
546    let c = crate::util::fast_random();
547    let d = crate::util::fast_random();
548
549    format!("{a:016x}-{b:016x}-{c:016x}-{d:016x}")
550}
551
552#[cfg(test)]
553mod tests {
554    use super::*;
555    use http_body_util::BodyExt;
556
557    use futures_util::{future, stream, TryStreamExt};
558    use tokio::{self, runtime};
559
560    #[test]
561    fn form_empty() {
562        let form = Form::new();
563
564        let rt = runtime::Builder::new_current_thread()
565            .enable_all()
566            .build()
567            .expect("new rt");
568        let body = form.stream().into_data_stream();
569        let s = body.map_ok(|try_c| try_c.to_vec()).try_concat();
570
571        let out = rt.block_on(s);
572        assert!(out.unwrap().is_empty());
573    }
574
575    #[test]
576    fn stream_to_end() {
577        let mut form = Form::new()
578            .part(
579                "reader1",
580                Part::body(boxed_stream(stream::once(future::ready::<
581                    Result<Frame<Bytes>, crate::error::BoxError>,
582                >(Ok(
583                    Frame::data(Bytes::from_static(b"part1")),
584                ))))),
585            )
586            .part("key1", Part::text("value1"))
587            .part("key2", Part::text("value2").mime(mime::IMAGE_BMP))
588            .part(
589                "reader2",
590                Part::body(boxed_stream(stream::once(future::ready::<
591                    Result<Frame<Bytes>, crate::error::BoxError>,
592                >(Ok(
593                    Frame::data(Bytes::from_static(b"part2")),
594                ))))),
595            )
596            .part("key3", Part::text("value3").file_name("filename"));
597        form.inner.boundary = "boundary".to_string();
598        let expected = "--boundary\r\n\
599             Content-Disposition: form-data; name=\"reader1\"\r\n\r\n\
600             part1\r\n\
601             --boundary\r\n\
602             Content-Disposition: form-data; name=\"key1\"\r\n\r\n\
603             value1\r\n\
604             --boundary\r\n\
605             Content-Disposition: form-data; name=\"key2\"\r\n\
606             Content-Type: image/bmp\r\n\r\n\
607             value2\r\n\
608             --boundary\r\n\
609             Content-Disposition: form-data; name=\"reader2\"\r\n\r\n\
610             part2\r\n\
611             --boundary\r\n\
612             Content-Disposition: form-data; name=\"key3\"; filename=\"filename\"\r\n\r\n\
613             value3\r\n--boundary--\r\n";
614        let rt = runtime::Builder::new_current_thread()
615            .enable_all()
616            .build()
617            .expect("new rt");
618        let body = form.stream().into_data_stream();
619        let s = body.map(|try_c| try_c.map(|r| r.to_vec())).try_concat();
620
621        let out = rt.block_on(s).unwrap();
622        // These prints are for debug purposes in case the test fails
623        println!(
624            "START REAL\n{}\nEND REAL",
625            std::str::from_utf8(&out).unwrap()
626        );
627        println!("START EXPECTED\n{expected}\nEND EXPECTED");
628        assert_eq!(std::str::from_utf8(&out).unwrap(), expected);
629    }
630
631    #[test]
632    fn stream_to_end_with_header() {
633        let mut part = Part::text("value2").mime(mime::IMAGE_BMP);
634        let mut headers = HeaderMap::new();
635        headers.insert("Hdr3", "/a/b/c".parse().unwrap());
636        part = part.headers(headers);
637        let mut form = Form::new().part("key2", part);
638        form.inner.boundary = "boundary".to_string();
639        let expected = "--boundary\r\n\
640                        Content-Disposition: form-data; name=\"key2\"\r\n\
641                        Content-Type: image/bmp\r\n\
642                        hdr3: /a/b/c\r\n\
643                        \r\n\
644                        value2\r\n\
645                        --boundary--\r\n";
646        let rt = runtime::Builder::new_current_thread()
647            .enable_all()
648            .build()
649            .expect("new rt");
650        let body = form.stream().into_data_stream();
651        let s = body.map(|try_c| try_c.map(|r| r.to_vec())).try_concat();
652
653        let out = rt.block_on(s).unwrap();
654        // These prints are for debug purposes in case the test fails
655        println!(
656            "START REAL\n{}\nEND REAL",
657            std::str::from_utf8(&out).unwrap()
658        );
659        println!("START EXPECTED\n{expected}\nEND EXPECTED");
660        assert_eq!(std::str::from_utf8(&out).unwrap(), expected);
661    }
662
663    #[test]
664    fn correct_content_length() {
665        // Setup an arbitrary data stream
666        let stream_data = b"just some stream data";
667        let stream_len = stream_data.len();
668        let stream_data = stream_data
669            .chunks(3)
670            .map(|c| Ok::<_, crate::error::BoxError>(Frame::data(Bytes::from(c))));
671        let the_stream = futures_util::stream::iter(stream_data);
672
673        let bytes_data = b"some bytes data".to_vec();
674        let bytes_len = bytes_data.len();
675
676        let stream_part = Part::body_with_length(boxed_stream(the_stream), stream_len as u64);
677        let body_part = Part::bytes(bytes_data);
678
679        // A simple check to make sure we get the configured body length
680        assert_eq!(stream_part.value_len().unwrap(), stream_len as u64);
681
682        // Make sure it delegates to the underlying body if length is not specified
683        assert_eq!(body_part.value_len().unwrap(), bytes_len as u64);
684    }
685
686    #[test]
687    fn header_percent_encoding() {
688        let name = "start%'\"\r\nßend";
689        let field = Part::text("");
690
691        assert_eq!(
692            PercentEncoding::PathSegment.encode_headers(name, &field.meta),
693            &b"Content-Disposition: form-data; name*=utf-8''start%25'%22%0D%0A%C3%9Fend"[..]
694        );
695
696        assert_eq!(
697            PercentEncoding::AttrChar.encode_headers(name, &field.meta),
698            &b"Content-Disposition: form-data; name*=utf-8''start%25%27%22%0D%0A%C3%9Fend"[..]
699        );
700    }
701}
client_util/request/multipart.rs

client_util/request/
multipart.rs
