legible 0.4.3

A Rust port of Mozilla's Readability.js for extracting readable content from web pages
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163

//! Functions to determine if a document is probably readerable.

use crate::constants::regexps;
use crate::dom::{build_match_string, get_tag_name, has_tag_name};
use crate::options::ReaderableOptions;
use crate::scoring::is_probably_visible;
use dom_query::{Document, Node, NodeId};
use hashbrown::HashSet;

/// Check if a document is probably readerable without parsing the whole thing.
///
/// This is a quick heuristic check to determine if [`parse()`](crate::parse)
/// is likely to succeed. If you want to check readability before parsing, use
/// [`Document`] to avoid parsing the HTML twice.
///
/// The function scores paragraph-like elements based on their text length, ignoring
/// elements that match unlikely candidate patterns (sidebars, navigation, etc.).
///
/// # Arguments
///
/// * `html` - The HTML content to check
/// * `options` - Optional [`ReaderableOptions`] to customize the check
///
/// # Returns
///
/// `true` if the document appears to contain readable article content, `false` otherwise.
///
/// # Example
///
/// ```rust
/// use legible::is_probably_readerable;
///
/// let html = r#"
///     <html><body>
///         <article>
///             <p>This is a substantial article with enough content
///             to be considered readable by the algorithm.</p>
///         </article>
///     </body></html>
/// "#;
///
/// if is_probably_readerable(html, None) {
///     println!("Document is likely readerable");
/// }
/// ```
pub fn is_probably_readerable(html: &str, options: Option<ReaderableOptions>) -> bool {
    let doc = Document::from(html);
    is_probably_readerable_doc(&doc, options)
}

pub(crate) fn is_probably_readerable_doc(
    doc: &Document,
    options: Option<ReaderableOptions>,
) -> bool {
    let options = options.unwrap_or_default();

    let mut score = 0.0;
    let mut seen_div_ids: HashSet<NodeId> = HashSet::new();

    // Reusable buffer for match_string to avoid allocations per node
    let mut match_string_buf = String::with_capacity(128);

    for node in doc.root().descendants_it().filter(|node| node.is_element()) {
        let tag_name = get_tag_name(&node).unwrap_or_default();
        if matches!(&*tag_name, "P" | "PRE" | "ARTICLE") {
            if score_readerable_node(&node, &options, &mut score, &mut match_string_buf) {
                return true;
            }
            continue;
        }

        if tag_name == "BR"
            && let Some(parent) = node.parent()
            && has_tag_name(&parent, "div")
            && seen_div_ids.insert(parent.id)
            && score_readerable_node(&parent, &options, &mut score, &mut match_string_buf)
        {
            return true;
        }
    }

    false
}

fn score_readerable_node(
    node: &Node<'_>,
    options: &ReaderableOptions,
    score: &mut f64,
    match_string_buf: &mut String,
) -> bool {
    // Check visibility (no match_string needed)
    if !is_probably_visible(node) {
        return false;
    }

    // Build match_string lazily — only needed for regex check
    build_match_string(node, match_string_buf);

    // Use RegexSet for single-pass matching of both patterns
    let candidate_matches = regexps::CANDIDATE_FILTER_SET.matches(match_string_buf);
    if candidate_matches.matched(0) && !candidate_matches.matched(1) {
        return false;
    }

    // Check if li > p (skip list item paragraphs)
    let mut parent = node.parent();
    while let Some(p) = parent {
        if has_tag_name(&p, "li") {
            return false;
        }
        parent = p.parent();
    }

    // Check text content length
    let text_length = node.normalized_char_count();

    if text_length < options.min_content_length {
        return false;
    }

    // Add to score based on content length
    *score += ((text_length - options.min_content_length) as f64).sqrt();

    *score > options.min_score
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_short_content_not_readerable() {
        let html = "<html><body><p>Short</p></body></html>";
        assert!(!is_probably_readerable(html, None));
    }

    #[test]
    fn test_long_content_is_readerable() {
        // Need sqrt(text_len - 140) > 20, so text_len > 540
        let long_text = "a".repeat(600);
        let html = format!("<html><body><p>{}</p></body></html>", long_text);
        assert!(is_probably_readerable(&html, None));
    }

    #[test]
    fn test_unlikely_candidates_ignored() {
        // Even long content is rejected if it has unlikely candidate class
        let long_text = "a".repeat(600);
        let html = format!(
            "<html><body><p class=\"sidebar\">{}</p></body></html>",
            long_text
        );
        assert!(!is_probably_readerable(&html, None));
    }

    #[test]
    fn test_article_tag_helps() {
        // Same scoring rules apply - article tag helps collect nodes but doesn't change scoring
        let text = "a".repeat(600);
        let html = format!("<html><body><article>{}</article></body></html>", text);
        assert!(is_probably_readerable(&html, None));
    }
}