docling 1.66.0

//! Email (`.eml` / Outlook `.msg`) backend — a port of docling's
//! `EmailDocumentBackend` (#251 for `.msg`, docling#3873).
//!
//! The subject becomes the document title; `From:`/`To:`/`Date:` headers become
//! text paragraphs; the body (preferring `text/plain`) is split into paragraphs
//! on blank lines. All emitted text is HTML/underscore-escaped like docling-core
//! (so `<a@b>` renders as `&lt;a@b&gt;`). A CFB-magic input is an Outlook
//! `.msg`: it projects onto RFC 822 (see [`super::msg`]) and flows through the
//! **same** parse below, so `.msg` and `.eml` output match by construction —
//! docling's own architecture for the format. `list_attachments` (opt-in,
//! docling's `EmailBackendOptions.list_attachments`) appends an `Attachments`
//! section listing names and content types; payload bytes are never embedded.

use mail_parser::{Address, Message, MessageParser};

use crate::backend::markdown::escape_text;
use crate::backend::DeclarativeBackend;
use crate::error::ConversionError;
use crate::source::SourceDocument;
use docling_core::{DoclingDocument, Node};

pub struct EmailBackend {
    /// Append an `Attachments` section (names + content types, never the
    /// payload) — docling's opt-in `list_attachments`.
    pub list_attachments: bool,
}

impl DeclarativeBackend for EmailBackend {
    fn convert(&self, source: &SourceDocument) -> Result<DoclingDocument, ConversionError> {
        // Outlook .msg (CFB magic): project to RFC 822 first; the projection
        // also carries the attachment labels straight from MAPI.
        let projected = crate::backend::cfb::CompoundFile::detect(&source.bytes)
            .then(|| super::msg::project(&source.bytes))
            .flatten();
        let raw: &[u8] = projected.as_ref().map_or(&source.bytes, |p| &p.rfc822);
        let msg = MessageParser::default()
            .parse(raw)
            .ok_or_else(|| ConversionError::Parse("email: could not parse message".into()))?;
        let mut doc = DoclingDocument::new(&source.name);

        if let Some(subject) = msg.subject().map(header_safe).filter(|s| !s.is_empty()) {
            doc.push(Node::Heading {
                level: 1,
                text: escape_text(&subject),
            });
        }
        for (label, addrs) in [("From", msg.from()), ("To", msg.to())] {
            let text = format_addresses(addrs);
            if !text.is_empty() {
                doc.push(Node::Paragraph {
                    text: escape_text(&format!("{label}: {text}")),
                });
            }
        }
        if let Some(date) = msg.date() {
            // Python docling formats via datetime.isoformat(), which spells
            // UTC as "+00:00" — mail-parser's RFC 3339 uses "Z". Align.
            let date = date.to_rfc3339().replace('Z', "+00:00");
            doc.push(Node::Paragraph {
                text: escape_text(&format!("Date: {date}")),
            });
        }
        for para in body_paragraphs(&msg) {
            doc.push(Node::Paragraph {
                text: escape_text(&para),
            });
        }
        if self.list_attachments {
            let labels = match &projected {
                Some(p) => p.attachment_labels.clone(),
                None => eml_attachment_labels(&msg),
            };
            if !labels.is_empty() {
                // docling adds the heading at level 2 under the title → "###".
                doc.push(Node::Heading {
                    level: 3,
                    text: "Attachments".into(),
                });
                for label in labels {
                    doc.push(Node::ListItem {
                        ordered: false,
                        number: 1,
                        first_in_list: false,
                        text: escape_text(&label),
                        level: 0,
                        marker: None,
                        location: None,
                        dclx: None,
                        href: None,
                        layer: None,
                    });
                }
            }
        }
        Ok(doc)
    }
}

/// Attachment display labels for a parsed `.eml`: `name (type/subtype)`,
/// falling back to `attachment-N` for nameless parts — docling's
/// `_get_attachment_labels`.
fn eml_attachment_labels(msg: &Message) -> Vec<String> {
    use mail_parser::MimeHeaders;
    msg.attachments()
        .enumerate()
        .map(|(i, part)| {
            // Header-derived, so single-line like every header (docling#4242).
            let name = part
                .attachment_name()
                .map(header_safe)
                .filter(|s| !s.is_empty())
                .unwrap_or_else(|| format!("attachment-{}", i + 1));
            match part.content_type() {
                Some(ct) => {
                    let mime = match ct.subtype() {
                        Some(sub) => format!("{}/{sub}", ct.ctype()),
                        None => ct.ctype().to_string(),
                    };
                    format!("{name} ({})", header_safe(&mime))
                }
                None => name,
            }
        })
        .collect()
}

/// docling's `_header_safe` (2.128, docling#4242): a header value collapsed
/// to one line of single-spaced text — `" ".join(value.split())`, which
/// covers every line terminator `str.splitlines()` knows and unfolds a
/// folded header to one space — so a decoded RFC 2047 encoded-word cannot
/// stand up a forged header line in the rendered document.
fn header_safe(value: &str) -> String {
    value.split_whitespace().collect::<Vec<_>>().join(" ")
}

/// docling's `_quote_display_name`: a rendered display name holding an RFC
/// 5322 special (`()<>[]:;@,\"` — `.` excepted, it parses back unquoted and
/// is common in real names) is quoted, with `\` and `"` escaped, so
/// `Name <email>` parses back as itself; a non-ASCII name stays readable
/// text (where `formataddr` would RFC 2047-encode it).
fn quote_display_name(name: &str) -> String {
    if name.chars().any(|c| {
        matches!(
            c,
            '(' | ')' | '<' | '>' | '[' | ']' | ':' | ';' | '@' | ',' | '"' | '\\'
        )
    }) {
        let escaped: String = name
            .chars()
            .flat_map(|c| {
                if matches!(c, '"' | '\\') {
                    vec!['\\', c]
                } else {
                    vec![c]
                }
            })
            .collect();
        format!("\"{escaped}\"")
    } else {
        name.to_string()
    }
}

/// `"Name <email>"` per address (or bare `email`), joined with `", "`.
fn format_addresses(addr: Option<&Address>) -> String {
    let Some(addr) = addr else {
        return String::new();
    };
    addr.iter()
        .filter_map(|a| {
            let name = a.name().map(header_safe).filter(|s| !s.is_empty());
            let email = a.address().map(str::trim).filter(|s| !s.is_empty());
            match (name, email) {
                (Some(n), Some(e)) => Some(format!("{} <{e}>", quote_display_name(&n))),
                (None, Some(e)) => Some(e.to_string()),
                _ => None,
            }
        })
        .collect::<Vec<_>>()
        .join(", ")
}

/// Body paragraphs (split on blank lines), preferring `text/plain`. CRLF and
/// a lone CR are normalised to `\n` first (docling#4248, 2.129), so a
/// CR-only body still splits into paragraphs.
fn body_paragraphs(msg: &Message) -> Vec<String> {
    let re = cached_regex!(r"\n\s*\n+");
    let split = |text: &str, out: &mut Vec<String>| {
        let text = text.replace("\r\n", "\n").replace('\r', "\n");
        for p in re.split(text.trim()) {
            let p = p.trim();
            if !p.is_empty() {
                out.push(p.to_string());
            }
        }
    };
    let mut out = Vec::new();
    let plain = msg.text_body_count();
    if plain > 0 {
        for i in 0..plain {
            if let Some(t) = msg.body_text(i) {
                split(&t, &mut out);
            }
        }
        return out;
    }
    // No plain text — fall back to the raw HTML body as text (the test corpus is
    // plain-text only; full HTML→Markdown of email bodies is a later refinement).
    for i in 0..msg.html_body_count() {
        if let Some(t) = msg.body_html(i) {
            split(&t, &mut out);
        }
    }
    out
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::format::InputFormat;

    #[test]
    fn title_headers_escaped_and_body_split() {
        let eml = "From: Alice <a@x.com>\r\nTo: Bob <b@y.com>\r\nSubject: Hi\r\n\
                   Content-Type: text/plain\r\n\r\nLine one.\r\n\r\nLine two.\r\n";
        let src = SourceDocument::from_bytes("m", InputFormat::Email, eml.as_bytes().to_vec());
        let md = EmailBackend {
            list_attachments: false,
        }
        .convert(&src)
        .unwrap()
        .export_to_markdown();
        // angle brackets HTML-escaped; body split into separate paragraphs.
        assert_eq!(
            md.trim(),
            "# Hi\n\nFrom: Alice &lt;a@x.com&gt;\n\nTo: Bob &lt;b@y.com&gt;\n\nLine one.\n\nLine two."
        );
    }

    /// #251: `list_attachments` appends the section with `name (type)` labels
    /// for `.eml` too — the docling label format, payload never embedded.
    /// docling#4242/#4248 (2.128–2.129): a display name with RFC 5322 specials
    /// is quoted (with `"` and `\` escaped), header values are collapsed to
    /// one line, and a CR-only body still splits into paragraphs.
    #[test]
    fn display_names_and_bodies_follow_docling_2129() {
        assert_eq!(quote_display_name("Doe, John"), "\"Doe, John\"");
        assert_eq!(quote_display_name("Ann \"Q\" Lee"), "\"Ann \\\"Q\\\" Lee\"");
        assert_eq!(quote_display_name("J. R. Hartley"), "J. R. Hartley");
        assert_eq!(quote_display_name("Zoë Müller"), "Zoë Müller");
        assert_eq!(header_safe("  two\r\n  lines  "), "two lines");
        let eml = "From: \"Doe, John\" <j@x.com>\r\nTo: b@y.com\r\nSubject: Re:\r\n folded\r\n\
                   Content-Type: text/plain\r\n\r\nOne.\r\rTwo.\r";
        let src = SourceDocument::from_bytes("m.eml", InputFormat::Email, eml.as_bytes().to_vec());
        let md = EmailBackend {
            list_attachments: false,
        }
        .convert(&src)
        .unwrap()
        .export_to_markdown();
        assert!(
            md.starts_with("# Re: folded\n\nFrom: \"Doe, John\" &lt;j@x.com&gt;\n"),
            "{md}"
        );
        assert!(md.ends_with("One.\n\nTwo.\n"), "{md}");
    }

    #[test]
    fn eml_list_attachments_appends_labels() {
        let eml = concat!(
            "From: A <a@x.com>\r\n",
            "To: B <b@y.com>\r\n",
            "Subject: S\r\n",
            "MIME-Version: 1.0\r\n",
            "Content-Type: multipart/mixed; boundary=\"bb\"\r\n\r\n",
            "--bb\r\nContent-Type: text/plain\r\n\r\nBody.\r\n",
            "--bb\r\nContent-Type: text/plain\r\n",
            "Content-Disposition: attachment; filename=\"note.txt\"\r\n\r\n",
            "hi\r\n--bb--\r\n",
        );
        let src = SourceDocument::from_bytes("m", crate::InputFormat::Email, eml.into());
        let md = EmailBackend {
            list_attachments: true,
        }
        .convert(&src)
        .unwrap()
        .export_to_markdown();
        assert!(md.contains("### Attachments"), "{md}");
        assert!(md.contains("- note.txt (text/plain)"), "{md}");
        let md_off = EmailBackend {
            list_attachments: false,
        }
        .convert(&src)
        .unwrap()
        .export_to_markdown();
        assert!(!md_off.contains("Attachments"), "{md_off}");
    }
}