eml_codec/text/
misc_token.rs

1#[cfg(feature = "arbitrary")]
2use arbitrary::Arbitrary;
3use bounded_static::ToStatic;
4use nom::{
5    branch::alt,
6    bytes::complete::{tag, take_while1},
7    combinator::{map, opt},
8    multi::{many0, many1, separated_list0},
9    sequence::delimited,
10    IResult, Parser,
11};
12use std::borrow::Cow;
13#[cfg(feature = "tracing")]
14use tracing::warn;
15
16use crate::i18n::ContainsUtf8;
17use crate::print::{print_seq, Formatter, Print, ToStringFromPrint};
18use crate::text::{
19    ascii,
20    encoding::{self, encoded_word, encoded_word_plain},
21    quoted::{quoted_string, QuotedString, QuotedStringChars},
22    utf8::take_utf8_while1,
23    whitespace::{cfws, fws, is_obs_no_ws_ctl},
24    words::{atom, is_vchar, mime_atom, Atom, MIMEAtom, MIMEAtomChars},
25};
26#[cfg(feature = "arbitrary")]
27use crate::{
28    arbitrary_utils::{
29        arbitrary_string_nonempty_where, arbitrary_vec_nonempty, arbitrary_whitespace_nonempty,
30    },
31    fuzz_eq::FuzzEq,
32};
33use eml_codec_derives::instrument_input;
34
35#[derive(Clone, ContainsUtf8, Debug, PartialEq, Default, ToStatic, ToStringFromPrint)]
36#[cfg_attr(feature = "arbitrary", derive(FuzzEq))]
37pub struct PhraseList<'a>(pub Vec<Phrase<'a>>); // must be nonempty
38
39#[cfg(feature = "arbitrary")]
40impl<'a> Arbitrary<'a> for PhraseList<'a> {
41    fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result<Self> {
42        Ok(Self(arbitrary_vec_nonempty(u)?))
43    }
44}
45
46/// A comma-separated list of phrases. Handles the obsolete syntax:
47///
48/// obs-phrase-list =   [phrase / CFWS] *("," [phrase / CFWS])
49///
50/// where phrase-list (not defined explicitly in the RFC, but used in keywords):
51///
52/// phrase-list     = phrase *("," phrase)
53///
54/// We return an option to represent the empty-list case of the obsolete syntax;
55/// and in turn, a `PhraseList` is always non-empty.
56#[instrument_input("tracing")]
57pub fn phrase_list(input: &[u8]) -> IResult<&[u8], Option<PhraseList<'_>>> {
58    let (input, phrases_opt) =
59        separated_list0(tag(","), alt((map(phrase, Some), map(opt(cfws), |_| None))))(input)?;
60    let phrases: Vec<Phrase> = phrases_opt.into_iter().flatten().collect();
61    if phrases.is_empty() {
62        Ok((input, None))
63    } else {
64        Ok((input, Some(PhraseList(phrases))))
65    }
66}
67impl<'a> Print for PhraseList<'a> {
68    fn print(&self, fmt: &mut impl Formatter) {
69        print_seq(fmt, &self.0, |fmt| {
70            fmt.write_bytes(b",");
71            fmt.write_fws()
72        })
73    }
74}
75
76#[derive(Clone, ContainsUtf8, Debug, PartialEq, ToStatic, ToStringFromPrint)]
77#[cfg_attr(feature = "arbitrary", derive(Arbitrary, FuzzEq))]
78pub enum MIMEWord<'a> {
79    Quoted(QuotedString<'a>),
80    Atom(MIMEAtom<'a>),
81}
82impl Default for MIMEWord<'static> {
83    fn default() -> Self {
84        Self::Atom(MIMEAtom::default())
85    }
86}
87#[instrument_input("tracing")]
88pub fn mime_word(input: &[u8]) -> IResult<&[u8], MIMEWord<'_>> {
89    alt((
90        map(quoted_string, MIMEWord::Quoted),
91        map(mime_atom, MIMEWord::Atom),
92    ))(input)
93}
94
95impl<'a> MIMEWord<'a> {
96    pub fn chars<'b>(&'b self) -> MIMEWordChars<'a, 'b> {
97        match self {
98            MIMEWord::Quoted(q) => MIMEWordChars::Quoted(q.chars()),
99            MIMEWord::Atom(a) => MIMEWordChars::Atom(a.chars()),
100        }
101    }
102}
103impl<'a> Print for MIMEWord<'a> {
104    fn print(&self, fmt: &mut impl Formatter) {
105        match self {
106            MIMEWord::Quoted(q) => q.print(fmt),
107            MIMEWord::Atom(a) => a.print(fmt),
108        }
109    }
110}
111
112#[derive(Clone)]
113pub enum MIMEWordChars<'a, 'b> {
114    Quoted(QuotedStringChars<'a, 'b>),
115    Atom(MIMEAtomChars<'a, 'b>),
116}
117
118impl<'a, 'b> Iterator for MIMEWordChars<'a, 'b> {
119    type Item = char;
120    fn next(&mut self) -> Option<Self::Item> {
121        match self {
122            MIMEWordChars::Quoted(q) => q.next(),
123            MIMEWordChars::Atom(a) => a.next(),
124        }
125    }
126}
127
128#[derive(Clone, ContainsUtf8, Debug, PartialEq, ToStatic, ToStringFromPrint)]
129#[cfg_attr(feature = "arbitrary", derive(Arbitrary, FuzzEq))]
130pub enum Word<'a> {
131    Quoted(QuotedString<'a>),
132    Atom(Atom<'a>),
133}
134
135impl<'a> Print for Word<'a> {
136    fn print(&self, fmt: &mut impl Formatter) {
137        match self {
138            Word::Quoted(q) => q.print(fmt),
139            Word::Atom(a) => a.print(fmt),
140        }
141    }
142}
143
144impl<'a> Word<'a> {
145    pub fn chars<'b>(&'b self) -> WordChars<'a, 'b> {
146        match self {
147            Word::Quoted(q) => WordChars::Quoted(q.chars()),
148            Word::Atom(a) => WordChars::Atom(a.0.chars()),
149        }
150    }
151}
152
153impl<'a> TryFrom<&'a str> for Word<'a> {
154    type Error = (); // TODO
155    fn try_from(s: &'a str) -> Result<Self, Self::Error> {
156        if let Ok(a) = Atom::try_from(s) {
157            return Ok(Word::Atom(a));
158        }
159        let qs = QuotedString::try_from(s)?;
160        Ok(Word::Quoted(qs))
161    }
162}
163
164#[derive(Clone)]
165pub enum WordChars<'a, 'b> {
166    Quoted(QuotedStringChars<'a, 'b>),
167    Atom(std::str::Chars<'b>),
168}
169
170impl<'a, 'b> Iterator for WordChars<'a, 'b> {
171    type Item = char;
172    fn next(&mut self) -> Option<Self::Item> {
173        match self {
174            WordChars::Quoted(q) => q.next(),
175            WordChars::Atom(a) => a.next(),
176        }
177    }
178}
179
180/// Word
181///
182/// ```abnf
183///    word            =   atom / quoted-string
184/// ```
185#[instrument_input("tracing")]
186pub fn word(input: &[u8]) -> IResult<&[u8], Word<'_>> {
187    alt((map(quoted_string, Word::Quoted), map(atom, Word::Atom)))(input)
188}
189
190#[derive(Clone, ContainsUtf8, Debug, PartialEq, ToStatic, ToStringFromPrint)]
191#[cfg_attr(feature = "arbitrary", derive(FuzzEq))]
192pub enum PhraseToken<'a> {
193    // Word MUST NOT be a Word::Atom that represents an encoded
194    // word, ie of the form =?..?..?..?=
195    Word(Word<'a>),
196    Encoded(encoding::EncodedWord<'a>),
197}
198impl<'a> Print for PhraseToken<'a> {
199    fn print(&self, fmt: &mut impl Formatter) {
200        match self {
201            PhraseToken::Word(w) => w.print(fmt),
202            PhraseToken::Encoded(e) => e.print(fmt),
203        }
204    }
205}
206impl<'a> PhraseToken<'a> {
207    /// Returns the text string represented by this PhraseToken
208    pub fn data(&self) -> String {
209        match self {
210            Self::Word(w) => w.chars().collect(),
211            Self::Encoded(e) => e.data(),
212        }
213    }
214}
215#[cfg(feature = "arbitrary")]
216impl<'a> Arbitrary<'a> for PhraseToken<'a> {
217    fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result<Self> {
218        if u.arbitrary()? {
219            let w: Word<'_> = u.arbitrary()?;
220            // As a coarse-grained measure, reject any atom that contains '=?'
221            // to avoid confusion with Encoded tokens
222            if let Word::Atom(a) = &w {
223                if a.0.find("=?").is_some() {
224                    return Err(arbitrary::Error::IncorrectFormat);
225                }
226            }
227            Ok(PhraseToken::Word(w))
228        } else {
229            Ok(PhraseToken::Encoded(u.arbitrary()?))
230        }
231    }
232}
233
234/// A part of a phrase or obs-phrase
235#[instrument_input("tracing")]
236pub fn phrase_token(input: &[u8]) -> IResult<&[u8], PhraseToken<'_>> {
237    alt((
238        // NOTE: try encoded words first because they can also be parsed as words.
239        map(
240            encoded_word(encoding::Context::Phrase),
241            PhraseToken::Encoded,
242        ),
243        map(word, PhraseToken::Word),
244        // "obs-phrase" allows periods while "phrase" does not.
245        // We could have a dedicated `Dot` constructor to `PhraseToken`
246        // to represent them, and later decide how `Dot` should be printed
247        // (note: printing must not use the obs- syntax!)
248        // Here, we use a different approach and directly parse naked dots
249        // as the AST for `"."` (note the quotes), which is allowed in the
250        // non-obs- syntax, thus ensuring that this AST can be safely
251        // printed as-is.
252        map(
253            delimited(opt(cfws), tag(&[ascii::PERIOD][..]), opt(cfws)),
254            |_| {
255                PhraseToken::Word(Word::Quoted(QuotedString(vec![Cow::Owned(
256                    ".".to_string(),
257                )])))
258            },
259        ),
260    ))(input)
261}
262
263// Must be a non-empty list.
264#[derive(Clone, ContainsUtf8, Debug, PartialEq, ToStatic, ToStringFromPrint)]
265pub struct Phrase<'a>(pub Vec<PhraseToken<'a>>);
266
267impl<'a> Print for Phrase<'a> {
268    fn print(&self, fmt: &mut impl Formatter) {
269        print_seq(fmt, &self.0, Formatter::write_fws)
270    }
271}
272
273impl<'a, 'b> From<&'b [&'a str]> for Phrase<'a> {
274    /// A phrase represents a sequence of space-separated words. This builds a
275    /// phrase from a list of such words.
276    ///
277    /// Note: due to constraints of the phrase syntax, consecutive non-ascii
278    /// words passed to `from` are joined together (separated with a space) into
279    /// a single "phrase word".
280    ///
281    /// Guarantee: `Phrase::from(words).data().join(" ") == words.join(" ")`
282    fn from(mut words: &'b [&'a str]) -> Self {
283        let mut toks = vec![];
284        let mut to_encode: Vec<&str> = vec![];
285
286        let push_encoded = |to_encode: &mut Vec<&str>, toks: &mut Vec<_>| {
287            if !to_encode.is_empty() {
288                toks.push(PhraseToken::Encoded(encoding::EncodedWord::from_chars(
289                    to_encode.join(" ").chars(),
290                )));
291                to_encode.clear();
292            }
293        };
294
295        while let [w, ws @ ..] = words {
296            // A `Word` can contain UTF8 text thanks to RFC6532, but we avoid
297            // relying on this RFC if we can as it does not seems to be well
298            // supported everywhere. Here we use an encoded word instead to
299            // represent non-ascii text.
300            if w.is_ascii() {
301                if let Ok(w) = Word::try_from(*w) {
302                    push_encoded(&mut to_encode, &mut toks);
303                    toks.push(PhraseToken::Word(w));
304                    words = ws;
305                    continue;
306                }
307            }
308            to_encode.push(w);
309            words = ws;
310        }
311        push_encoded(&mut to_encode, &mut toks);
312        Self(toks)
313    }
314}
315impl From<&[String]> for Phrase<'static> {
316    fn from(words: &[String]) -> Self {
317        use bounded_static::ToBoundedStatic;
318        let words: Vec<&str> = words.as_ref().iter().map(|s| s.as_str()).collect();
319        Phrase::from(words.as_slice()).to_static()
320    }
321}
322impl<'a> From<&'a str> for Phrase<'a> {
323    fn from(word: &'a str) -> Self {
324        Phrase::from(&[word][..])
325    }
326}
327impl From<String> for Phrase<'static> {
328    fn from(word: String) -> Self {
329        use bounded_static::ToBoundedStatic;
330        Phrase::from(&[word][..]).to_static()
331    }
332}
333
334impl<'a> Phrase<'a> {
335    /// Return a list of the space-separated "words" this phrase represents.
336    pub fn data(&self) -> Vec<String> {
337        self.0.iter().map(|tok| tok.data()).collect()
338    }
339
340    // Merges consecutive Encoded tokens
341    #[cfg(feature = "arbitrary")]
342    fn normalize(&self) -> Self {
343        let mut v = Vec::new();
344        for tok in &self.0 {
345            match (v.last_mut(), tok) {
346                (Some(PhraseToken::Encoded(ref mut e1)), PhraseToken::Encoded(e2)) => {
347                    e1.0.extend(e2.0.clone())
348                }
349                (_, tok) => v.push(tok.clone()),
350            }
351        }
352        Self(v)
353    }
354}
355#[cfg(feature = "arbitrary")]
356impl<'a> Arbitrary<'a> for Phrase<'a> {
357    fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result<Self> {
358        Ok(Self(arbitrary_vec_nonempty(u)?))
359    }
360}
361#[cfg(feature = "arbitrary")]
362impl<'a> FuzzEq for Phrase<'a> {
363    fn fuzz_eq(&self, other: &Self) -> bool {
364        self.normalize().0.fuzz_eq(&other.normalize().0)
365    }
366}
367
368/// Phrase (including obsolete syntax)
369///
370/// ```abnf
371///    phrase          =   1*(encoded-word / word) / obs-phrase
372///    obs-phrase      =   word *(word / "." / CFWS)
373/// ```
374///
375/// (encoded-word comes from RFC2047)
376///
377/// The grammar above is equivalent to the following, which is
378/// what we parse:
379///
380/// ```abnf
381///   phrase       =  1*phrase_token
382///   phrase_token =  encoded-word / word / ([CFWS] "." [CFWS])
383/// ```
384#[instrument_input("tracing")]
385pub fn phrase(input: &[u8]) -> IResult<&[u8], Phrase<'_>> {
386    let (input, phrase) = map(many1(phrase_token), Phrase)(input)?;
387    Ok((input, phrase))
388}
389
390#[derive(Debug, PartialEq, Clone, ToStatic)]
391pub struct UtextToken<'a> {
392    txt: Cow<'a, str>,
393    obs: bool,
394}
395
396/// Compatible unstructured input
397///
398/// ```abnf
399/// obs-utext       =   %d0 / obs-NO-WS-CTL / VCHAR
400/// ```
401/// and also non-ascii UTF-8 text, following RFC6532.
402///
403/// The parser result records which parts of the input
404/// were using the obsolete syntax (i.e. not VCHAR).
405///
406/// Parses a single run of either obsolete or non-obsolete
407/// characters.
408fn obs_utext_token<'a>(input: &'a [u8]) -> IResult<&'a [u8], UtextToken<'a>> {
409    alt((
410        take_utf8_while1(is_vchar).map(|s| UtextToken { txt: s, obs: false }),
411        take_while1(|c| is_obs_no_ws_ctl(c) || c == ascii::NULL)
412            // SAFETY: from the line above we know that `s` contains ASCII bytes
413            // (they satisfy either is_obs_no_ws_ctl or are NULL).
414            .map(|s| unsafe { str::from_utf8_unchecked(s) })
415            .map(|s| UtextToken {
416                txt: Cow::Borrowed(s),
417                obs: true,
418            }),
419    ))(input)
420}
421
422#[derive(Debug, PartialEq, Copy, Clone, ToStatic)]
423pub enum UnstrTxtKind {
424    Txt, // non-space text
425    Obs, // non-space text using obsolete characters
426    Fws, // whitespace
427}
428
429#[derive(PartialEq, Clone, Debug, ToStatic)]
430#[cfg_attr(feature = "arbitrary", derive(FuzzEq))]
431pub enum UnstrToken<'a> {
432    Encoded(encoding::EncodedWord<'a>),
433    // `Plain` MUST NOT contain text that represents an encoded word,
434    // ie. of the form =?..?..?..?=
435    #[cfg_attr(feature = "arbitrary", fuzz_eq(use_eq))]
436    Plain(Cow<'a, str>, UnstrTxtKind),
437}
438
439impl<'a> UnstrToken<'a> {
440    pub(crate) fn from_plain(s: &'a str, kind: UnstrTxtKind) -> Self {
441        Self::Plain(Cow::Borrowed(s), kind)
442    }
443
444    fn from_utext(tok: UtextToken<'a>) -> Self {
445        if tok.obs {
446            Self::Plain(tok.txt, UnstrTxtKind::Obs)
447        } else {
448            Self::Plain(tok.txt, UnstrTxtKind::Txt)
449        }
450    }
451}
452impl<'a> ContainsUtf8 for UnstrToken<'a> {
453    fn contains_utf8(&self) -> bool {
454        match self {
455            UnstrToken::Encoded(_) => false,
456            UnstrToken::Plain(s, _) => s.contains_utf8(),
457        }
458    }
459}
460impl<'a> Print for UnstrToken<'a> {
461    fn print(&self, fmt: &mut impl Formatter) {
462        match self {
463            UnstrToken::Encoded(e) => e.print(fmt),
464            UnstrToken::Plain(txt, UnstrTxtKind::Txt) => fmt.write_bytes(txt.as_bytes()),
465            UnstrToken::Plain(_, UnstrTxtKind::Obs) =>
466                // skip obsolete parts
467                {}
468            UnstrToken::Plain(txt, UnstrTxtKind::Fws) => fmt.write_fws_bytes(txt.as_bytes()),
469        }
470    }
471}
472#[cfg(feature = "arbitrary")]
473impl<'a> Arbitrary<'a> for UnstrToken<'a> {
474    fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result<Self> {
475        // XXX we do not generate `Obs` tokens because those are dropped at printing time.
476        // this is somewhat of a hack.
477        match u.int_in_range(0..=2)? {
478            0 => Ok(UnstrToken::Encoded(u.arbitrary()?)),
479            1 => {
480                let txt = arbitrary_string_nonempty_where(u, is_vchar, 'X')?;
481                // As a coarse-grained measure, reject text that contains '=?' to avoid confusion with
482                // Encoded tokens
483                if txt.find("=?").is_some() {
484                    return Err(arbitrary::Error::IncorrectFormat);
485                }
486                Ok(UnstrToken::Plain(txt.into(), UnstrTxtKind::Txt))
487            }
488            2 => {
489                let txt = arbitrary_whitespace_nonempty(u)?;
490                Ok(UnstrToken::Plain(txt.into(), UnstrTxtKind::Fws))
491            }
492            _ => unreachable!(),
493        }
494    }
495}
496
497// Invariant:
498// - Encoded and Plain(_, Txt) tokens are always separated by whitespace
499//   (encoded words must be separated from other words by whitespace)
500// - There must not be whitespace token in between two Encoded tokens
501//   (whitespace in between encoded words is meaningless and is ignored during parsing)
502#[derive(Debug, PartialEq, Clone, ToStatic, ToStringFromPrint)]
503pub struct Unstructured<'a>(pub Vec<UnstrToken<'a>>);
504
505impl<'a> Print for Unstructured<'a> {
506    fn print(&self, fmt: &mut impl Formatter) {
507        for i in 0..self.0.len() {
508            let tok = &self.0[i];
509
510            // consecutive encoded tokens must be separated by whitespace
511            if i > 0 {
512                if let (UnstrToken::Encoded(_), UnstrToken::Encoded(_)) = (&self.0[i - 1], tok) {
513                    fmt.write_fws()
514                }
515            }
516
517            tok.print(fmt)
518        }
519    }
520}
521
522impl<'a> Unstructured<'a> {
523    pub fn to_string_keep_obs(&self) -> String {
524        let mut s = String::new();
525        for tok in &self.0 {
526            match tok {
527                UnstrToken::Encoded(e) => s.push_str(&e.to_string()),
528                UnstrToken::Plain(txt, _) => s.push_str(txt),
529            }
530        }
531        s
532    }
533
534    // Merges consecutive tokens of the same kind.
535    // Used to define fuzz_eq.
536    #[cfg(feature = "arbitrary")]
537    fn fuzz_eq_normalize(&self) -> Unstructured<'static> {
538        use bounded_static::ToBoundedStatic;
539        let mut v: Vec<UnstrToken<'static>> = Vec::new();
540        for tok in &self.0 {
541            match (v.last_mut(), tok) {
542                (Some(UnstrToken::Plain(s1, k1)), UnstrToken::Plain(s2, k2)) if k1 == k2 => {
543                    s1.to_mut().push_str(s2)
544                }
545                (Some(UnstrToken::Encoded(e1)), UnstrToken::Encoded(e2)) => {
546                    e1.0.extend(e2.to_static().0)
547                }
548                _ => v.push(tok.to_static()),
549            }
550        }
551        Unstructured(v)
552    }
553}
554impl<'a> ContainsUtf8 for Unstructured<'a> {
555    fn contains_utf8(&self) -> bool {
556        self.0.contains_utf8()
557    }
558}
559
560#[cfg(feature = "arbitrary")]
561impl<'a> FuzzEq for Unstructured<'a> {
562    fn fuzz_eq(&self, other: &Self) -> bool {
563        self.fuzz_eq_normalize()
564            .0
565            .fuzz_eq(&other.fuzz_eq_normalize().0)
566    }
567}
568
569#[cfg(feature = "arbitrary")]
570impl<'a> Arbitrary<'a> for Unstructured<'a> {
571    fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result<Self> {
572        enum Kind {
573            Encoded,
574            Wsp,
575            Txt,
576        }
577        fn k(tok: &UnstrToken<'_>) -> Kind {
578            match tok {
579                UnstrToken::Encoded(_) => Kind::Encoded,
580                UnstrToken::Plain(_, UnstrTxtKind::Fws) => Kind::Wsp,
581                UnstrToken::Plain(_, _) => Kind::Txt,
582            }
583        }
584
585        let mut v: Vec<UnstrToken> = Vec::new();
586        let mut before_last = None;
587        let mut last = None;
588        for _ in 0..u.arbitrary_len::<UnstrToken>()? {
589            let tok: UnstrToken = u.arbitrary()?;
590            match (&before_last, &last, k(&tok)) {
591                // invariant: no whitespace between encoded tokens
592                (Some(Kind::Encoded), Some(Kind::Wsp), Kind::Encoded) |
593                // invariant: encoded and text must be separated by whitespace
594                (_, Some(Kind::Encoded), Kind::Txt) | (_, Some(Kind::Txt), Kind::Encoded) => {
595                    return Err(arbitrary::Error::IncorrectFormat)
596                },
597
598                // consecutive Txt or Wsp nodes should be treated as one when
599                // tracking "the" previous token kind
600                (_, Some(Kind::Wsp), Kind::Wsp) | (_, Some(Kind::Txt), Kind::Txt) =>
601                    (),
602                (_, _, ktok) => {
603                    before_last = last;
604                    last = Some(ktok);
605                }
606            };
607            v.push(tok)
608        }
609        Ok(Unstructured(v))
610    }
611}
612
613/// Unstructured header field body
614///
615/// Grammar from the RFC:
616/// ```abnf
617/// unstructured    =   (*([FWS] VCHAR_SEQ) *WSP) / obs-unstruct
618/// obs-unstruct    =   *((*CR 1*(obs_utext / FWS)) / 1*LF) *CR   (cf errata)
619/// ```
620/// + RFC 2047 (MIME pt3) encoded words
621///
622/// However, in our relaxed parsing of line endings in whitespace::obs_crlf,
623/// bare CR or LF are treated as line endings and thus cannot also be part of
624/// the obs-unstruct syntax.
625///
626/// We thus choose to parse obs-unstruct minus the bare CR and LF, which
627/// corresponds to (also adding RFC2047 encoded words):
628///
629/// our-obs-unstruct    =   *(encoded-words / obs_utext / FWS)
630///
631/// This does not match the RFC but seems to better match real-world practices.
632#[instrument_input("tracing")]
633pub fn unstructured(input: &[u8]) -> IResult<&[u8], Unstructured<'_>> {
634    let (input, r) = many0(alt((
635        map(encoded_word_plain(encoding::Context::Unstructured), |w| {
636            vec![UnstrToken::Encoded(w)]
637        }),
638        map(obs_utext_token, |tok| vec![UnstrToken::from_utext(tok)]),
639        map(fws, |v| {
640            v.into_iter()
641                .map(|s| UnstrToken::from_plain(s, UnstrTxtKind::Fws))
642                .collect()
643        }),
644    )))(input)?;
645
646    Ok((input, Unstructured(r.into_iter().flatten().collect())))
647}
648
649#[cfg(test)]
650mod tests {
651    use super::*;
652    use crate::print::tests::print_to_vec;
653    use crate::text::charset::EmailCharset;
654    use crate::text::encoding::{EncodedWord, EncodedWordToken, QuotedChunk, QuotedWord};
655
656    #[test]
657    fn test_word_from() {
658        assert_eq!(Word::try_from("abc"), Ok(Word::Atom(Atom("abc".into()))),);
659
660        assert_eq!(Word::try_from("").unwrap().to_string(), "\"\"",);
661
662        assert_eq!(Word::try_from("a b").unwrap().to_string(), "\"a b\"",);
663    }
664
665    #[test]
666    fn test_phrase_from() {
667        let check = |words: &[&str]| {
668            let p = Phrase::from(words);
669            assert_eq!(words.join(" "), p.data().join(" "))
670        };
671
672        check(&["abc", "d ef", "éé", "à"]);
673        check(&["abc", "", "", "à"]);
674        check(&["éè", "a", "ö", "  ï", "w"]);
675
676        assert_eq!(
677            Phrase::from(&["a", "", "b c", "é", "ï"][..]).to_string(),
678            "a \"\" \"b c\" =?UTF-8?Q?=C3=A9_=C3=AF?=",
679        );
680    }
681
682    #[test]
683    fn test_phrase() {
684        assert_eq!(
685            print_to_vec(phrase(b"hello world").unwrap().1),
686            b"hello world".to_vec(),
687        );
688        // XXX: would we rather want this to be reprinted as "salut le monde"?
689        // (since the quotes are unnecessary in this case)
690        assert_eq!(
691            print_to_vec(phrase(b"salut \"le\" monde").unwrap().1),
692            b"salut \"le\" monde".to_vec(),
693        );
694
695        let (rest, parsed) = phrase(b"fin\r\n du\r\nmonde").unwrap();
696        assert_eq!(rest, &b"\r\nmonde"[..]);
697        assert_eq!(&print_to_vec(parsed), b"fin du");
698
699        let (rest, parsed) = phrase(b"foo.bar").unwrap();
700        assert_eq!(rest, &b""[..]);
701        assert_eq!(&print_to_vec(parsed), b"foo \".\" bar");
702    }
703
704    #[test]
705    fn test_phrase_list() {
706        let (rest, parsed) = phrase_list(b",abc def,,   ,ghi").unwrap();
707        assert_eq!(rest, &b""[..]);
708        assert_eq!(&print_to_vec(parsed.as_ref().unwrap()), b"abc def, ghi");
709    }
710
711    #[test]
712    fn test_unstructured() {
713        let (rest, parsed) = unstructured(b"").unwrap();
714        assert_eq!(rest, &b""[..]);
715        assert_eq!(parsed, Unstructured(vec![]));
716
717        let (rest, parsed) = unstructured(b" \t").unwrap();
718        assert_eq!(rest, &b""[..]);
719        assert_eq!(
720            parsed,
721            Unstructured(vec![UnstrToken::Plain(" \t"[..].into(), UnstrTxtKind::Fws)])
722        );
723
724        let (rest, parsed) = unstructured(b"foo =?UTF-8?q?foo?=").unwrap();
725        assert_eq!(rest, &b""[..]);
726        assert_eq!(
727            parsed,
728            Unstructured(vec![
729                UnstrToken::Plain("foo"[..].into(), UnstrTxtKind::Txt),
730                UnstrToken::Plain(" "[..].into(), UnstrTxtKind::Fws),
731                UnstrToken::Encoded(EncodedWord(vec![EncodedWordToken::Quoted(QuotedWord {
732                    enc: EmailCharset::utf8(),
733                    chunks: vec![QuotedChunk::Safe(b"foo"[..].into())],
734                })]))
735            ])
736        );
737
738        // RFC 2047 specifies that encoded words MUST be separated from other
739        // words by whitespace. otherwise, we parse them as normal text...
740        let (rest, parsed) = unstructured(b"foo=?UTF-8?q?foo?=").unwrap();
741        assert_eq!(rest, &b""[..]);
742        assert_eq!(
743            parsed,
744            Unstructured(vec![UnstrToken::Plain(
745                "foo=?UTF-8?q?foo?="[..].into(),
746                UnstrTxtKind::Txt
747            ),])
748        );
749
750        // trailing FWS is allowed
751        let (rest, parsed) = unstructured(b"foo\r\n\t").unwrap();
752        assert_eq!(rest, &b""[..]);
753        assert_eq!(
754            parsed,
755            Unstructured(vec![
756                UnstrToken::Plain("foo"[..].into(), UnstrTxtKind::Txt),
757                UnstrToken::Plain("\t"[..].into(), UnstrTxtKind::Fws),
758            ])
759        );
760    }
761}
eml_codec/text/misc_token.rs

eml_codec/text/
misc_token.rs
