Skip to main content

fetchium_core/extract/
layer2.rs

1//! CEP Layer 2: Readability-style extraction using `lol_html` streaming rewriter.
2//!
3//! PRD SS16: "HTTP + Readability" — ~8ms, ~10MB RAM, for article pages.
4//! Uses lol_html to stream through HTML, stripping boilerplate in a single
5//! pass without building a full DOM tree.
6
7use crate::extract::boilerplate::{self, clean_text, MIN_CONTENT_LENGTH};
8use crate::extract::layer1::estimate_tokens;
9use crate::extract::{ContentMetadata, ExtractedContent};
10use crate::types::CepLayer;
11use lol_html::{element, rewrite_str, RewriteStrSettings};
12use once_cell::sync::Lazy;
13use regex::Regex;
14use tracing::debug;
15
16/// Tags to remove entirely (element and all children).
17const REMOVE_TAGS: &[&str] = &[
18    "script", "style", "nav", "footer", "aside", "noscript", "svg", "iframe", "form", "button",
19    "input", "select", "textarea",
20];
21
22/// Extract content using lol_html streaming rewriter (Layer 2).
23///
24/// This method is more aggressive at stripping boilerplate than Layer 1.
25/// It processes HTML in a single streaming pass, making it memory-efficient
26/// for large pages.
27///
28/// QADD pre-filtering with `boilerplate::strip_heavy_tags` is applied before
29/// the lol_html pass to further reduce the amount of data the streamer must
30/// process.
31pub fn extract(html: &str, url: &str) -> ExtractedContent {
32    // QADD: strip heavy tags before DOM parsing to reduce parse cost
33    let stripped_html = boilerplate::strip_heavy_tags(html);
34
35    let title = extract_title_simple(&stripped_html);
36    let metadata = extract_metadata_simple(&stripped_html);
37
38    let cleaned_html = strip_boilerplate(&stripped_html);
39    let text = html_to_text(&cleaned_html);
40    let cleaned = clean_text(&text);
41    let tokens = estimate_tokens(&cleaned);
42
43    debug!(
44        "Layer2: extracted {} chars, ~{} tokens from {}",
45        cleaned.len(),
46        tokens,
47        url
48    );
49
50    ExtractedContent {
51        title,
52        text: cleaned,
53        layer_used: CepLayer::Layer2,
54        tokens,
55        metadata,
56    }
57}
58
59/// Check if Layer 2 extraction produced sufficient content.
60pub fn is_sufficient(content: &ExtractedContent) -> bool {
61    content.text.len() >= MIN_CONTENT_LENGTH
62}
63
64/// Strip boilerplate tags using lol_html streaming rewriter.
65fn strip_boilerplate(html: &str) -> String {
66    let mut selectors = Vec::new();
67
68    for tag in REMOVE_TAGS {
69        selectors.push(element!(tag, |el| {
70            el.remove();
71            Ok(())
72        }));
73    }
74
75    selectors.push(element!("[role='navigation']", |el| {
76        el.remove();
77        Ok(())
78    }));
79    selectors.push(element!("[role='banner']", |el| {
80        el.remove();
81        Ok(())
82    }));
83    selectors.push(element!("[role='contentinfo']", |el| {
84        el.remove();
85        Ok(())
86    }));
87    selectors.push(element!("[aria-hidden='true']", |el| {
88        el.remove();
89        Ok(())
90    }));
91    selectors.push(element!(
92        ".sidebar, .nav, .menu, .footer, .header, .ads, .ad, .advertisement",
93        |el| {
94            el.remove();
95            Ok(())
96        }
97    ));
98
99    match rewrite_str(
100        html,
101        RewriteStrSettings {
102            element_content_handlers: selectors,
103            ..RewriteStrSettings::default()
104        },
105    ) {
106        Ok(result) => result,
107        Err(_) => html.to_string(),
108    }
109}
110
111/// Convert cleaned HTML to plain text, preserving paragraph structure.
112fn html_to_text(html: &str) -> String {
113    static TAG_RE: Lazy<Regex> = Lazy::new(|| Regex::new(r"<[^>]+>").expect("valid regex"));
114    static ENTITY_RE: Lazy<Regex> = Lazy::new(|| {
115        Regex::new(r"&(amp|lt|gt|quot|apos|nbsp|#\d+|#x[0-9a-fA-F]+);").expect("valid regex")
116    });
117
118    let text = html
119        .replace("<br>", "\n")
120        .replace("<br/>", "\n")
121        .replace("<br />", "\n")
122        .replace("</p>", "\n\n")
123        .replace("</div>", "\n")
124        .replace("</li>", "\n")
125        .replace("</tr>", "\n")
126        .replace("</h1>", "\n\n")
127        .replace("</h2>", "\n\n")
128        .replace("</h3>", "\n\n")
129        .replace("</h4>", "\n")
130        .replace("</h5>", "\n")
131        .replace("</h6>", "\n")
132        .replace("</blockquote>", "\n");
133
134    let text = TAG_RE.replace_all(&text, "");
135
136    let text = ENTITY_RE.replace_all(&text, |caps: &regex::Captures| match &caps[1] {
137        "amp" => "&".to_string(),
138        "lt" => "<".to_string(),
139        "gt" => ">".to_string(),
140        "quot" => "\"".to_string(),
141        "apos" => "'".to_string(),
142        "nbsp" => " ".to_string(),
143        s if s.starts_with('#') => {
144            let num = if let Some(hex) = s.strip_prefix("#x") {
145                u32::from_str_radix(hex, 16).ok()
146            } else {
147                s[1..].parse::<u32>().ok()
148            };
149            num.and_then(char::from_u32)
150                .map(|c| c.to_string())
151                .unwrap_or_default()
152        }
153        _ => String::new(),
154    });
155
156    text.to_string()
157}
158
159/// Simple title extraction without full DOM parsing.
160fn extract_title_simple(html: &str) -> String {
161    static TITLE_RE: Lazy<Regex> =
162        Lazy::new(|| Regex::new(r"(?i)<title[^>]*>(.*?)</title>").expect("valid regex"));
163
164    TITLE_RE
165        .captures(html)
166        .and_then(|c| c.get(1))
167        .map(|m| m.as_str().trim().to_string())
168        .unwrap_or_default()
169}
170
171/// Simple metadata extraction using regex (no DOM).
172fn extract_metadata_simple(html: &str) -> ContentMetadata {
173    static DESC_RE: Lazy<Regex> = Lazy::new(|| {
174        Regex::new(r#"(?i)<meta[^>]*name=["']description["'][^>]*content=["']([^"']*)["']"#)
175            .expect("valid regex")
176    });
177    static AUTHOR_RE: Lazy<Regex> = Lazy::new(|| {
178        Regex::new(r#"(?i)<meta[^>]*name=["']author["'][^>]*content=["']([^"']*)["']"#)
179            .expect("valid regex")
180    });
181    static OG_DESC_RE: Lazy<Regex> = Lazy::new(|| {
182        Regex::new(r#"(?i)<meta[^>]*property=["']og:description["'][^>]*content=["']([^"']*)["']"#)
183            .expect("valid regex")
184    });
185
186    ContentMetadata {
187        description: DESC_RE
188            .captures(html)
189            .and_then(|c| c.get(1))
190            .map(|m| m.as_str().to_string())
191            .or_else(|| {
192                OG_DESC_RE
193                    .captures(html)
194                    .and_then(|c| c.get(1))
195                    .map(|m| m.as_str().to_string())
196            }),
197        author: AUTHOR_RE
198            .captures(html)
199            .and_then(|c| c.get(1))
200            .map(|m| m.as_str().to_string()),
201        published_date: None,
202        language: None,
203        content_type: "text/html".to_string(),
204    }
205}
206
207#[cfg(test)]
208mod tests {
209    use super::*;
210
211    #[test]
212    fn extract_article_page() {
213        let html = r#"
214        <html>
215        <head><title>Article Title</title></head>
216        <body>
217            <nav><a href="/">Home</a><a href="/about">About</a></nav>
218            <script>var tracking = true;</script>
219            <article>
220                <h1>Main Heading</h1>
221                <p>First paragraph with substantial content that should
222                   definitely be extracted and preserved in the output
223                   because it forms the core of the article text.</p>
224                <p>Second paragraph with more details about the topic
225                   that provides additional context and information
226                   for the reader to understand the full picture.</p>
227            </article>
228            <footer>Copyright 2026</footer>
229        </body>
230        </html>
231        "#;
232
233        let result = extract(html, "https://example.com/article");
234        assert_eq!(result.title, "Article Title");
235        assert!(result.text.contains("First paragraph"));
236        assert!(result.text.contains("Second paragraph"));
237        assert!(!result.text.contains("tracking"));
238        assert!(!result.text.contains("Copyright"));
239        assert_eq!(result.layer_used, CepLayer::Layer2);
240    }
241
242    #[test]
243    fn html_entity_decoding() {
244        let text = html_to_text("<p>Tom &amp; Jerry &lt;3 &quot;cheese&quot;</p>");
245        assert!(text.contains("Tom & Jerry"));
246        assert!(text.contains("<3"));
247        assert!(text.contains("\"cheese\""));
248    }
249
250    #[test]
251    fn strip_scripts_and_styles() {
252        let html = r#"
253        <html><body>
254        <script>alert('xss')</script>
255        <style>.foo { color: red; }</style>
256        <p>Real content here</p>
257        </body></html>
258        "#;
259        let cleaned = strip_boilerplate(html);
260        assert!(!cleaned.contains("alert"));
261        assert!(!cleaned.contains("color: red"));
262        assert!(cleaned.contains("Real content"));
263    }
264}