langmail 0.8.0

Email preprocessing for LLMs
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70

use langmail::preprocess;

const RAW: &[u8] = include_bytes!("fixtures/linear-login.eml");

/// Characters that clean_invisible_characters strips.
const INVISIBLE_CHARS: &[char] = &[
    '\u{034F}', // COMBINING GRAPHEME JOINER
    '\u{200B}', // ZERO WIDTH SPACE
    '\u{200C}', // ZERO WIDTH NON-JOINER
    '\u{200D}', // ZERO WIDTH JOINER
    '\u{FEFF}', // ZERO WIDTH NO-BREAK SPACE (BOM)
    '\u{00AD}', // SOFT HYPHEN
    '\u{2007}', // FIGURE SPACE
];

#[test]
fn no_invisible_characters() {
    let output = preprocess(RAW).unwrap();
    for c in INVISIBLE_CHARS {
        assert!(
            !output.body.contains(*c),
            "body contains invisible char U+{:04X}",
            *c as u32
        );
    }
}

#[test]
fn no_whitespace_only_lines() {
    let output = preprocess(RAW).unwrap();
    for (i, line) in output.body.lines().enumerate() {
        assert!(
            line.is_empty() || !line.trim().is_empty(),
            "line {} is whitespace-only: {:?}",
            i + 1,
            line
        );
    }
}

#[test]
fn no_excessive_empty_lines() {
    let output = preprocess(RAW).unwrap();
    assert!(
        !output.body.contains("\n\n\n"),
        "body contains 3+ consecutive newlines:\n{}",
        output.body
    );
}

#[test]
fn content_preserved() {
    let output = preprocess(RAW).unwrap();
    assert!(
        output.body.contains("New login to Max"),
        "missing 'New login to Max'"
    );
    assert!(
        output.body.contains("Platform: Max Desktop on macOS"),
        "missing platform info"
    );
    assert!(
        output.body.contains("Munich, BY, DE"),
        "missing location info"
    );
    assert!(
        output.body.contains("Security & access"),
        "missing security link text"
    );
}