legible 0.4.2

A Rust port of Mozilla's Readability.js for extracting readable content from web pages
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172

//! Functions to determine if a document is probably readerable.

use crate::constants::regexps;
use crate::dom::{build_match_string, get_tag_name};
use crate::options::ReaderableOptions;
use crate::scoring::is_probably_visible;
use crate::selectors::SELECTORS;
use dom_query::{Document, Node, NodeId};
use hashbrown::HashSet;

/// Check if a document is probably readerable without parsing the whole thing.
///
/// This is a quick heuristic check to determine if [`parse()`](crate::parse)
/// is likely to succeed. If you want to check readability before parsing, use
/// [`Document`] to avoid parsing the HTML twice.
///
/// The function scores paragraph-like elements based on their text length, ignoring
/// elements that match unlikely candidate patterns (sidebars, navigation, etc.).
///
/// # Arguments
///
/// * `html` - The HTML content to check
/// * `options` - Optional [`ReaderableOptions`] to customize the check
///
/// # Returns
///
/// `true` if the document appears to contain readable article content, `false` otherwise.
///
/// # Example
///
/// ```rust
/// use legible::is_probably_readerable;
///
/// let html = r#"
///     <html><body>
///         <article>
///             <p>This is a substantial article with enough content
///             to be considered readable by the algorithm.</p>
///         </article>
///     </body></html>
/// "#;
///
/// if is_probably_readerable(html, None) {
///     println!("Document is likely readerable");
/// }
/// ```
pub fn is_probably_readerable(html: &str, options: Option<ReaderableOptions>) -> bool {
    let doc = Document::from(html);
    is_probably_readerable_doc(&doc, options)
}

pub(crate) fn is_probably_readerable_doc(
    doc: &Document,
    options: Option<ReaderableOptions>,
) -> bool {
    let options = options.unwrap_or_default();

    // Collect actual nodes: p, pre, article
    let mut nodes: Vec<Node<'_>> = doc
        .select_matcher(&SELECTORS.p_pre_article)
        .nodes()
        .to_vec();

    // Track seen IDs to avoid duplicates when adding parent divs
    let mut seen_ids: HashSet<NodeId> = nodes.iter().map(|n| n.id).collect();

    // Add parent divs of br elements (with deduplication)
    for br in doc.select_matcher(&SELECTORS.div_br).nodes().iter() {
        if let Some(parent) = br.parent()
            && seen_ids.insert(parent.id)
        {
            nodes.push(parent);
        }
    }

    let mut score = 0.0;

    // Reusable buffer for match_string to avoid allocations per node
    let mut match_string_buf = String::with_capacity(128);

    // Iterate directly over the collected nodes
    for node in nodes.iter() {
        // Check visibility
        if !is_probably_visible(node) {
            continue;
        }

        // Build match_string for regex - reuse buffer to avoid allocations
        build_match_string(node, &mut match_string_buf);

        // Use RegexSet for single-pass matching of both patterns
        let candidate_matches = regexps::CANDIDATE_FILTER_SET.matches(&match_string_buf);
        if candidate_matches.matched(0) && !candidate_matches.matched(1) {
            continue;
        }

        // Check if li > p (skip list item paragraphs)
        let is_li_p = {
            let mut parent = node.parent();
            let mut result = false;
            while let Some(p) = parent {
                if let Some(tag) = get_tag_name(&p)
                    && tag == "LI"
                {
                    result = true;
                    break;
                }
                parent = p.parent();
            }
            result
        };

        if is_li_p {
            continue;
        }

        // Check text content length
        let text = node.text();
        let text_length = text.trim().chars().count();

        if text_length < options.min_content_length {
            continue;
        }

        // Add to score based on content length
        score += ((text_length - options.min_content_length) as f64).sqrt();

        if score > options.min_score {
            return true;
        }
    }

    false
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_short_content_not_readerable() {
        let html = "<html><body><p>Short</p></body></html>";
        assert!(!is_probably_readerable(html, None));
    }

    #[test]
    fn test_long_content_is_readerable() {
        // Need sqrt(text_len - 140) > 20, so text_len > 540
        let long_text = "a".repeat(600);
        let html = format!("<html><body><p>{}</p></body></html>", long_text);
        assert!(is_probably_readerable(&html, None));
    }

    #[test]
    fn test_unlikely_candidates_ignored() {
        // Even long content is rejected if it has unlikely candidate class
        let long_text = "a".repeat(600);
        let html = format!(
            "<html><body><p class=\"sidebar\">{}</p></body></html>",
            long_text
        );
        assert!(!is_probably_readerable(&html, None));
    }

    #[test]
    fn test_article_tag_helps() {
        // Same scoring rules apply - article tag helps collect nodes but doesn't change scoring
        let text = "a".repeat(600);
        let html = format!("<html><body><article>{}</article></body></html>", text);
        assert!(is_probably_readerable(&html, None));
    }
}