mail_parser/store/
mod.rs

1/*
2 * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
3 *
4 * SPDX-License-Identifier: Apache-2.0 OR MIT
5 */
6
7mod str;
8mod table;
9
10pub(crate) use str::{Str, validated_string};
11pub(crate) use table::{Blank, Table};
12
13use crate::{DateTime, Encoding, Greeting, Protocol, TlsVersion};
14use std::{fmt, net::IpAddr};
15
16pub(crate) const NONE: u32 = u32::MAX;
17const FIRST_TEXT_ITEMS: usize = 16;
18const INLINE_IDS: usize = 4;
19const INLINE_PARAMS: usize = 2;
20const INLINE_CONTENT_TYPES: usize = 2;
21const INLINE_TEXT_ITEMS: usize = 8;
22const INLINE_ADDRESS_ITEMS: usize = 4;
23
24/// Index of a part in [`crate::Message::parts`], in document order across
25/// nested messages.
26pub type PartId = u32;
27
28/// Index of a message in [`crate::Message::messages`]; 0 is the root message.
29pub type MessageId = u32;
30
31#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
32pub(crate) struct Span {
33    pub(crate) start: u32,
34    pub(crate) len: u32,
35}
36
37impl Span {
38    pub(crate) fn range(self) -> std::ops::Range<usize> {
39        let start = self.start as usize;
40        start..start + self.len as usize
41    }
42}
43
44#[derive(Debug, Clone, Copy, PartialEq, Eq)]
45pub(crate) struct MessageEntry {
46    pub(crate) root: PartId,
47    pub(crate) parts_end: PartId,
48    pub(crate) source: u32,
49    pub(crate) container: PartId,
50    pub(crate) lists: u32,
51    pub(crate) text_len: u32,
52    pub(crate) html_len: u32,
53    pub(crate) attachments_len: u32,
54}
55
56impl Blank for MessageEntry {
57    const BLANK: MessageEntry = MessageEntry {
58        root: NONE,
59        parts_end: NONE,
60        source: 0,
61        container: NONE,
62        lists: 0,
63        text_len: 0,
64        html_len: 0,
65        attachments_len: 0,
66    };
67}
68
69#[derive(Debug, Clone)]
70pub(crate) struct Messages {
71    root: MessageEntry,
72    has_root: bool,
73    nested: Vec<MessageEntry>,
74}
75
76impl Default for Messages {
77    fn default() -> Self {
78        Messages::EMPTY
79    }
80}
81
82impl Messages {
83    pub(crate) const EMPTY: Messages = Messages {
84        root: MessageEntry::BLANK,
85        has_root: false,
86        nested: Vec::new(),
87    };
88
89    #[inline]
90    pub(crate) fn len(&self) -> usize {
91        usize::from(self.has_root) + self.nested.len()
92    }
93
94    #[inline]
95    pub(crate) fn get(&self, index: usize) -> Option<&MessageEntry> {
96        match index.checked_sub(1) {
97            None => Some(&self.root),
98            Some(index) => self.nested.get(index),
99        }
100    }
101
102    #[inline]
103    pub(crate) fn get_mut(&mut self, index: usize) -> Option<&mut MessageEntry> {
104        match index.checked_sub(1) {
105            None => Some(&mut self.root),
106            Some(index) => self.nested.get_mut(index),
107        }
108    }
109
110    pub(crate) fn first(&self) -> Option<&MessageEntry> {
111        Some(&self.root)
112    }
113
114    pub(crate) fn push(&mut self, entry: MessageEntry, first: usize) -> MessageId {
115        if !self.has_root {
116            self.root = entry;
117            self.has_root = true;
118            return 0;
119        }
120        if self.nested.capacity() == 0 {
121            self.nested.reserve_exact(first);
122        }
123        self.nested.push(entry);
124        self.nested.len() as MessageId
125    }
126
127    pub(crate) fn pop(&mut self) -> Option<MessageEntry> {
128        self.nested.pop()
129    }
130
131    pub(crate) fn clear(&mut self) {
132        self.root = MessageEntry::BLANK;
133        self.has_root = false;
134        self.nested.clear();
135    }
136
137    #[cfg(test)]
138    pub(crate) fn iter(&self) -> impl Iterator<Item = &MessageEntry> {
139        std::iter::once(&self.root)
140            .take(usize::from(self.has_root))
141            .chain(&self.nested)
142    }
143}
144
145#[derive(Debug, Clone, Copy, PartialEq, Eq)]
146#[repr(u8)]
147pub(crate) enum KindTag {
148    Text,
149    Html,
150    Binary,
151    InlineBinary,
152    Multipart,
153    Message,
154}
155
156pub(crate) mod flag {
157    pub(crate) const MISSING_DELIMITER: u16 = 1;
158    pub(crate) const FALLBACK_DELIMITER: u16 = 1 << 1;
159    pub(crate) const NO_BLANK_LINE: u16 = 1 << 2;
160    pub(crate) const UNTERMINATED: u16 = 1 << 3;
161    pub(crate) const NESTING_LIMIT: u16 = 1 << 4;
162    pub(crate) const LIMIT_REACHED: u16 = 1 << 5;
163    pub(crate) const MASK: u16 = 0xff;
164}
165
166pub(crate) mod role {
167    pub(crate) const IN_TEXT_BODY: u16 = 1 << 8;
168    pub(crate) const IN_HTML_BODY: u16 = 1 << 9;
169    pub(crate) const ATTACHMENT: u16 = 1 << 10;
170    pub(crate) const INLINE: u16 = 1 << 11;
171    pub(crate) const COPIED: u16 = 1 << 12;
172}
173
174#[derive(Debug, Clone, Copy, PartialEq, Eq)]
175pub(crate) struct PartEntry {
176    pub(crate) offset_header: u32,
177    pub(crate) offset_body: u32,
178    pub(crate) offset_end: u32,
179    pub(crate) headers: Span,
180    pub(crate) parent: PartId,
181    pub(crate) message: MessageId,
182    pub(crate) content_type: u32,
183    pub(crate) content_disposition: u32,
184    pub(crate) children: Span,
185    pub(crate) kind: KindTag,
186    pub(crate) encoding: Encoding,
187    pub(crate) flags: u16,
188}
189
190impl PartEntry {
191    pub(crate) const EMPTY: PartEntry = PartEntry {
192        offset_header: 0,
193        offset_body: 0,
194        offset_end: 0,
195        headers: Span { start: 0, len: 0 },
196        parent: NONE,
197        message: 0,
198        content_type: NONE,
199        content_disposition: NONE,
200        children: Span { start: 0, len: 0 },
201        kind: KindTag::Text,
202        encoding: Encoding::None,
203        flags: 0,
204    };
205
206    pub(crate) fn has_role(&self, role: u16) -> bool {
207        self.flags & role != 0
208    }
209}
210
211#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
212pub(crate) enum Value {
213    #[default]
214    Empty,
215    Text(Str),
216    TextList(Span),
217    Address(Span),
218    DateTime(DateTime),
219    ContentType(u32),
220    Received(u32),
221}
222
223pub(crate) const OTHER_NAME: u16 = u16::MAX;
224
225#[derive(Debug, Clone, Copy, PartialEq, Eq)]
226pub(crate) struct HeaderEntry {
227    pub(crate) offset_field: u32,
228    pub(crate) offset_start: u32,
229    pub(crate) offset_end: u32,
230    pub(crate) value: Value,
231    pub(crate) name: u16,
232    pub(crate) other_name: u32,
233}
234
235pub(crate) const GROUP_BIT: u32 = 1 << 31;
236
237#[derive(Debug, Clone, Copy, PartialEq, Eq)]
238pub(crate) struct AddressEntry {
239    pub(crate) name: Str,
240    pub(crate) address: Str,
241    pub(crate) group: u32,
242}
243
244impl Blank for AddressEntry {
245    const BLANK: AddressEntry = AddressEntry {
246        name: Str::NONE,
247        address: Str::NONE,
248        group: 0,
249    };
250}
251
252impl AddressEntry {
253    pub(crate) fn is_group(&self) -> bool {
254        self.group & GROUP_BIT != 0
255    }
256
257    pub(crate) fn members(&self) -> usize {
258        (self.group & !GROUP_BIT) as usize
259    }
260}
261
262#[derive(Debug, Clone, Copy, PartialEq, Eq)]
263pub(crate) struct ParamEntry {
264    pub(crate) name: Str,
265    pub(crate) value: Str,
266}
267
268impl Blank for ParamEntry {
269    const BLANK: ParamEntry = ParamEntry {
270        name: Str::NONE,
271        value: Str::NONE,
272    };
273}
274
275#[derive(Debug, Clone, Copy, PartialEq, Eq)]
276pub(crate) struct ContentTypeEntry {
277    pub(crate) ctype: Str,
278    pub(crate) subtype: Str,
279    pub(crate) params: Span,
280}
281
282impl Blank for ContentTypeEntry {
283    const BLANK: ContentTypeEntry = ContentTypeEntry {
284        ctype: Str::NONE,
285        subtype: Str::NONE,
286        params: Span { start: 0, len: 0 },
287    };
288}
289
290#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
291pub(crate) enum HostEntry {
292    #[default]
293    None,
294    Name(Str),
295    Ip(IpAddr),
296}
297
298#[derive(Debug, Clone, Copy, PartialEq, Eq)]
299pub(crate) struct ReceivedEntry {
300    pub(crate) from: HostEntry,
301    pub(crate) from_ip: Option<IpAddr>,
302    pub(crate) from_iprev: Str,
303    pub(crate) by: HostEntry,
304    pub(crate) for_: Str,
305    pub(crate) with: Option<Protocol>,
306    pub(crate) tls_version: Option<TlsVersion>,
307    pub(crate) tls_cipher: Str,
308    pub(crate) id: Str,
309    pub(crate) ident: Str,
310    pub(crate) helo: HostEntry,
311    pub(crate) helo_cmd: Option<Greeting>,
312    pub(crate) via: Str,
313    pub(crate) date: Option<DateTime>,
314}
315
316impl ReceivedEntry {
317    pub(crate) const EMPTY: ReceivedEntry = ReceivedEntry {
318        from: HostEntry::None,
319        from_ip: None,
320        from_iprev: Str::NONE,
321        by: HostEntry::None,
322        for_: Str::NONE,
323        with: None,
324        tls_version: None,
325        tls_cipher: Str::NONE,
326        id: Str::NONE,
327        ident: Str::NONE,
328        helo: HostEntry::None,
329        helo_cmd: None,
330        via: Str::NONE,
331        date: None,
332    };
333}
334
335impl Default for ReceivedEntry {
336    fn default() -> Self {
337        ReceivedEntry::EMPTY
338    }
339}
340
341#[derive(Debug, Clone, Default)]
342pub(crate) struct Scratch {
343    pub(crate) frames: Vec<crate::parser::Frame>,
344    pub(crate) pending: Vec<PartId>,
345    pub(crate) bytes: Vec<u8>,
346    pub(crate) text: String,
347    pub(crate) continuations: Vec<(Str, u32, Str)>,
348}
349
350impl Scratch {
351    const EMPTY: Scratch = Scratch {
352        frames: Vec::new(),
353        pending: Vec::new(),
354        bytes: Vec::new(),
355        text: String::new(),
356        continuations: Vec::new(),
357    };
358
359    fn clear(&mut self) {
360        self.frames.clear();
361        self.pending.clear();
362        self.bytes.clear();
363        self.text.clear();
364        self.continuations.clear();
365    }
366}
367
368#[derive(Debug, Clone)]
369pub(crate) struct SourceBuffer {
370    pub(crate) bytes: Box<[u8]>,
371    pub(crate) container: PartId,
372}
373
374#[derive(Debug, Clone, Default)]
375pub(crate) struct MessageData {
376    pub(crate) messages: Messages,
377    pub(crate) parts: Vec<PartEntry>,
378    pub(crate) headers: Vec<HeaderEntry>,
379    pub(crate) ids: Table<PartId, INLINE_IDS>,
380    pub(crate) address_items: Table<AddressEntry, INLINE_ADDRESS_ITEMS>,
381    pub(crate) params: Table<ParamEntry, INLINE_PARAMS>,
382    pub(crate) content_types: Table<ContentTypeEntry, INLINE_CONTENT_TYPES>,
383    pub(crate) text_items: Table<Str, INLINE_TEXT_ITEMS>,
384    pub(crate) received: Vec<ReceivedEntry>,
385    pub(crate) strings: String,
386    pub(crate) sources: Vec<SourceBuffer>,
387    pub(crate) scratch: Scratch,
388}
389
390impl MessageData {
391    pub(crate) const EMPTY: MessageData = MessageData {
392        messages: Messages::EMPTY,
393        parts: Vec::new(),
394        headers: Vec::new(),
395        ids: Table::EMPTY,
396        address_items: Table::EMPTY,
397        params: Table::EMPTY,
398        content_types: Table::EMPTY,
399        text_items: Table::EMPTY,
400        received: Vec::new(),
401        strings: String::new(),
402        sources: Vec::new(),
403        scratch: Scratch::EMPTY,
404    };
405
406    pub(crate) fn empty_message() -> Self {
407        let mut messages = Messages::EMPTY;
408        messages.push(
409            MessageEntry {
410                root: 0,
411                parts_end: 1,
412                ..MessageEntry::BLANK
413            },
414            0,
415        );
416        MessageData {
417            messages,
418            parts: vec![PartEntry::EMPTY],
419            ..MessageData::EMPTY
420        }
421    }
422
423    fn capacity_bytes(&self) -> usize {
424        [
425            vec_bytes(&self.messages.nested),
426            vec_bytes(&self.parts),
427            vec_bytes(&self.headers),
428            self.ids.capacity_bytes(),
429            self.address_items.capacity_bytes(),
430            self.params.capacity_bytes(),
431            self.content_types.capacity_bytes(),
432            self.text_items.capacity_bytes(),
433            vec_bytes(&self.received),
434            self.strings.capacity(),
435            vec_bytes(&self.sources),
436            self.sources.iter().map(|source| source.bytes.len()).sum(),
437            vec_bytes(&self.scratch.frames),
438            vec_bytes(&self.scratch.pending),
439            self.scratch.bytes.capacity(),
440            self.scratch.text.capacity(),
441            vec_bytes(&self.scratch.continuations),
442        ]
443        .into_iter()
444        .sum()
445    }
446
447    pub(crate) fn push_text_item(&mut self, item: Str) -> u32 {
448        let index = self.text_items.len() as u32;
449        self.text_items.push(item, FIRST_TEXT_ITEMS);
450        index
451    }
452
453    pub(crate) fn clear(&mut self) {
454        self.messages.clear();
455        self.parts.clear();
456        self.headers.clear();
457        self.ids.clear();
458        self.address_items.clear();
459        self.params.clear();
460        self.content_types.clear();
461        self.text_items.clear();
462        self.received.clear();
463        self.strings.clear();
464        self.sources.clear();
465        self.scratch.clear();
466    }
467
468    pub(crate) fn source<'a>(&'a self, raw: &'a [u8], source: u32) -> &'a [u8] {
469        match source.checked_sub(1) {
470            None => raw,
471            Some(index) => self
472                .sources
473                .get(index as usize)
474                .map_or(&[], |source| source.bytes.as_ref()),
475        }
476    }
477
478    pub(crate) fn source_container(&self, source: u32) -> Option<PartId> {
479        self.sources
480            .get(source.checked_sub(1)? as usize)
481            .map(|source| source.container)
482    }
483
484    pub(crate) fn message_source<'a>(&'a self, raw: &'a [u8], message: MessageId) -> &'a [u8] {
485        if message == 0 {
486            return raw;
487        }
488        self.messages
489            .get(message as usize)
490            .map_or(raw, |entry| self.source(raw, entry.source))
491    }
492}
493
494/// Reusable storage for [`crate::MessageParser::parse_with`].
495///
496/// A parsed message keeps its parts, headers and values in a handful of
497/// vectors. Parsing many messages one after the other (an mbox or Maildir
498/// import, classifier training, a batch of messages from a queue) can reuse
499/// those vectors instead of allocating new ones for every message: pass the
500/// buffers to `parse_with`, and when done with the message, get them back
501/// with [`crate::Message::into_buffers`]. Once the vectors have grown to
502/// fit the messages, parsing allocates nothing.
503///
504/// There is no hidden state: dropping a message instead of calling
505/// `into_buffers` only means the next parse allocates, and dropping the
506/// buffers frees the memory they hold.
507///
508/// The vectors never shrink on their own: they keep the capacity of the
509/// largest message parsed with them, so one very large message in a batch
510/// holds its memory for the rest of the loop. [`MessageBuffers::shrink_to`]
511/// bounds that memory between two parses.
512///
513/// ```
514/// use mail_parser::{MessageBuffers, MessageParser, mailbox::mbox::MessageIterator};
515///
516/// let mbox = b"From a@example.com Sat Jan  3 01:05:34 1996\n\
517/// Subject: first\n\nbody\n\
518/// From b@example.com Sat Jan  3 01:05:35 1996\n\
519/// Subject: second\n\nbody\n";
520///
521/// let parser = MessageParser::new();
522/// let mut buffers = MessageBuffers::new();
523/// let mut subjects = Vec::new();
524/// for entry in MessageIterator::new(&mbox[..]) {
525///     let entry = entry.expect("readable mbox");
526///     if let Some(message) = parser.parse_with(entry.contents(), &mut buffers) {
527///         subjects.push(message.subject().unwrap_or_default().to_string());
528///         buffers = message.into_buffers();
529///     }
530/// }
531/// assert_eq!(subjects, ["first", "second"]);
532/// ```
533#[derive(Clone, Default)]
534pub struct MessageBuffers {
535    pub(crate) data: MessageData,
536}
537
538impl fmt::Debug for MessageBuffers {
539    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
540        f.debug_struct("MessageBuffers").finish_non_exhaustive()
541    }
542}
543
544impl MessageBuffers {
545    /// Empty buffers; they grow to fit the messages parsed with them.
546    pub fn new() -> Self {
547        MessageBuffers::default()
548    }
549
550    /// Frees the memory held by the buffers when it exceeds `max_bytes`,
551    /// so that they hold at most `max_bytes` afterwards; the next parse then
552    /// allocates again. Call it between two parses, for example after
553    /// [`crate::Message::into_buffers`] in a batch loop, to keep one large
554    /// message from pinning its memory. It costs nothing when the buffers
555    /// are within the bound.
556    ///
557    /// ```
558    /// use mail_parser::{MessageBuffers, MessageParser};
559    ///
560    /// let parser = MessageParser::new();
561    /// let mut buffers = MessageBuffers::new();
562    /// for raw in [&b"Subject: small\n\nbody\n"[..], b"Subject: next\n\nbody\n"] {
563    ///     if let Some(message) = parser.parse_with(raw, &mut buffers) {
564    ///         assert!(message.subject().is_some());
565    ///         buffers = message.into_buffers();
566    ///         buffers.shrink_to(1 << 20);
567    ///     }
568    /// }
569    /// ```
570    pub fn shrink_to(&mut self, max_bytes: usize) {
571        if self.data.capacity_bytes() > max_bytes {
572            self.data = MessageData::default();
573        }
574    }
575}
576
577fn vec_bytes<T>(items: &Vec<T>) -> usize {
578    items.capacity() * size_of::<T>()
579}
580
581#[cfg(test)]
582mod tests {
583    use super::*;
584    use std::mem::size_of;
585
586    #[test]
587    fn root_message_stays_inline() {
588        let entry = |root| MessageEntry {
589            root,
590            ..MessageEntry::BLANK
591        };
592        let mut messages = Messages::default();
593        assert_eq!(messages.len(), 0);
594        assert_eq!(messages.iter().count(), 0);
595        assert_eq!(messages.get(0), Some(&MessageEntry::BLANK));
596        assert_eq!(messages.push(entry(1), 8), 0);
597        assert_eq!(messages.push(entry(2), 8), 1);
598        assert_eq!(messages.push(entry(3), 8), 2);
599        assert_eq!(messages.len(), 3);
600        assert_eq!(
601            messages.iter().map(|entry| entry.root).collect::<Vec<_>>(),
602            [1, 2, 3]
603        );
604        if let Some(nested) = messages.get_mut(1) {
605            nested.root = 7;
606        }
607        assert_eq!(messages.get(1).map(|entry| entry.root), Some(7));
608        assert_eq!(messages.get(3), None);
609        assert_eq!(messages.pop().map(|entry| entry.root), Some(3));
610        assert_eq!(messages.first().map(|entry| entry.root), Some(1));
611        messages.clear();
612        assert_eq!(messages.len(), 0);
613        assert_eq!(messages.push(entry(4), 8), 0);
614        assert_eq!(messages.len(), 1);
615    }
616
617    #[test]
618    fn buffers_shrink_above_the_bound() {
619        let mut raw = b"Content-Type: multipart/mixed; boundary=b\n\n".to_vec();
620        for index in 0..500 {
621            raw.extend_from_slice(format!("--b\nX-Index: {index}\n\npart {index}\n").as_bytes());
622        }
623        raw.extend_from_slice(b"--b--\n");
624        let parser = crate::MessageParser::new();
625        let mut buffers = MessageBuffers::new();
626        assert_eq!(buffers.data.capacity_bytes(), 0);
627        let message = parser.parse_with(&raw, &mut buffers).expect("parses");
628        buffers = message.into_buffers();
629        let held = buffers.data.capacity_bytes();
630        assert!(held >= 501 * size_of::<PartEntry>());
631        buffers.shrink_to(held);
632        assert_eq!(buffers.data.capacity_bytes(), held);
633        buffers.shrink_to(held - 1);
634        assert_eq!(buffers.data.capacity_bytes(), 0);
635    }
636
637    #[test]
638    fn empty_message_has_one_root_part() {
639        let data = MessageData::empty_message();
640        assert_eq!(data.messages.len(), 1);
641        assert_eq!(data.messages.first().map(|entry| entry.root), Some(0));
642        assert_eq!(data.parts, [PartEntry::EMPTY]);
643        assert_eq!(data.source_container(0), None);
644        assert_eq!(data.source_container(1), None);
645    }
646
647    #[test]
648    fn entry_sizes() {
649        assert_eq!(size_of::<Str>(), 8);
650        assert_eq!(size_of::<Value>(), 12);
651        assert_eq!(size_of::<MessageEntry>(), 32);
652        assert_eq!(size_of::<PartEntry>(), 48);
653        assert_eq!(size_of::<HeaderEntry>(), 32);
654        assert_eq!(size_of::<AddressEntry>(), 20);
655        assert_eq!(size_of::<ParamEntry>(), 16);
656        assert_eq!(size_of::<ContentTypeEntry>(), 24);
657        assert!(size_of::<ReceivedEntry>() <= 160);
658        assert!(size_of::<crate::parser::Frame>() <= 96);
659    }
660}
mail_parser/store/mod.rs

mail_parser/store/
mod.rs
