readabilityrs 0.1.4

A Rust port of Mozilla's Readability library for extracting article content from web pages
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132

//! Regression tests for issue #24: post-processing collapsed the indentation
//! inside `<pre>`/`<code>`, so extracted code listings came out flattened.

use readabilityrs::{Article, Readability, ReadabilityOptions};

const PADDING: &str = "This paragraph exists only to push the article over the \
    scoring threshold so the extractor keeps the section that holds the code \
    listing under test, padding padding padding padding padding.";

fn parse(body: &str) -> Article {
    parse_with(
        body,
        ReadabilityOptions {
            output_markdown: true,
            ..Default::default()
        },
    )
}

fn parse_with(body: &str, options: ReadabilityOptions) -> Article {
    let html = format!(
        "<html><head><title>Code</title></head><body><article>\
         <h1>Code</h1><p>{PADDING}</p>{body}<p>{PADDING}</p></article></body></html>"
    );

    Readability::new(&html, Some("https://example.com/a"), Some(options))
        .expect("parser construction")
        .parse()
        .expect("article extraction")
}

#[test]
fn test_pre_block_keeps_indentation_in_content_and_markdown() {
    let article = parse(
        "<pre tabindex=\"0\" class=\"chroma\"><code class=\"language-rust\">fn main() {\n\
         \x20   let x = 1;\n\
         \x20       deeper();\n\
         }\n</code></pre>",
    );

    let content = article.content.expect("content");
    assert!(content.contains("\n    let x = 1;\n"), "got: {content}");
    assert!(content.contains("\n        deeper();\n"), "got: {content}");

    let markdown = article.markdown_content.expect("markdown");
    assert!(markdown.contains("\n    let x = 1;\n"), "got: {markdown}");
    assert!(
        markdown.contains("\n        deeper();\n"),
        "got: {markdown}"
    );
}

#[test]
fn test_pre_block_keeps_blank_lines() {
    let article = parse("<pre><code>first();\n\n\n\nlast();\n</code></pre>");

    let content = article.content.expect("content");
    assert!(
        content.contains("first();\n\n\n\nlast();"),
        "got: {content}"
    );
}

#[test]
fn test_inline_code_keeps_internal_spacing() {
    let article = parse("<p>The literal <code>a    b</code> matters.</p>");

    let content = article.content.expect("content");
    assert!(content.contains("<code>a    b</code>"), "got: {content}");
}

#[test]
fn test_comment_inside_pre_does_not_end_the_listing() {
    let article = parse("<pre><code><!-- </pre> -->fn f() {\n    body();\n}\n</code></pre>");

    let content = article.content.expect("content");
    assert!(content.contains("\n    body();\n"), "got: {content}");
}

#[test]
fn test_title_removal_keeps_code_indentation() {
    let article = parse_with(
        "<pre><code>fn f() {\n\n\n    body();\n   \n    tail();\n}\n</code></pre>",
        ReadabilityOptions {
            output_markdown: true,
            remove_title_from_content: true,
            ..Default::default()
        },
    );

    let content = article.content.expect("content");
    assert!(
        !content.contains("<h1"),
        "title removal did not run: {content}"
    );
    assert!(
        content.contains("{\n\n\n    body();\n   \n"),
        "got: {content}"
    );
}

/// Title removal strips empty wrapper elements as well as whitespace, and that
/// loop is routed through the same splitter. An empty `<header>` written inside
/// a listing is text the reader typed, not a wrapper to drop.
#[test]
fn test_title_removal_keeps_empty_wrappers_inside_a_listing() {
    let article = parse_with(
        "<pre><header>   </header>\n    indented\n</pre>",
        ReadabilityOptions {
            output_markdown: true,
            remove_title_from_content: true,
            ..Default::default()
        },
    );

    let content = article.content.expect("content");
    assert!(
        !content.contains("<h1"),
        "title removal did not run: {content}"
    );
    assert!(content.contains("<header>   </header>"), "got: {content}");
    assert!(content.contains("\n    indented\n"), "got: {content}");
}

#[test]
fn test_prose_whitespace_still_collapses() {
    let article = parse("<p>ordinary    prose    spacing</p><pre>  kept  </pre>");

    let content = article.content.expect("content");
    assert!(content.contains("ordinary prose spacing"), "got: {content}");
    assert!(content.contains("<pre>  kept  </pre>"), "got: {content}");
}