fetchium_core/extract/
layer2.rs1use crate::extract::boilerplate::{self, clean_text, MIN_CONTENT_LENGTH};
8use crate::extract::layer1::estimate_tokens;
9use crate::extract::{ContentMetadata, ExtractedContent};
10use crate::types::CepLayer;
11use lol_html::{element, rewrite_str, RewriteStrSettings};
12use once_cell::sync::Lazy;
13use regex::Regex;
14use tracing::debug;
15
16const REMOVE_TAGS: &[&str] = &[
18 "script", "style", "nav", "footer", "aside", "noscript", "svg", "iframe", "form", "button",
19 "input", "select", "textarea",
20];
21
22pub fn extract(html: &str, url: &str) -> ExtractedContent {
32 let stripped_html = boilerplate::strip_heavy_tags(html);
34
35 let title = extract_title_simple(&stripped_html);
36 let metadata = extract_metadata_simple(&stripped_html);
37
38 let cleaned_html = strip_boilerplate(&stripped_html);
39 let text = html_to_text(&cleaned_html);
40 let cleaned = clean_text(&text);
41 let tokens = estimate_tokens(&cleaned);
42
43 debug!(
44 "Layer2: extracted {} chars, ~{} tokens from {}",
45 cleaned.len(),
46 tokens,
47 url
48 );
49
50 ExtractedContent {
51 title,
52 text: cleaned,
53 layer_used: CepLayer::Layer2,
54 tokens,
55 metadata,
56 }
57}
58
59pub fn is_sufficient(content: &ExtractedContent) -> bool {
61 content.text.len() >= MIN_CONTENT_LENGTH
62}
63
64fn strip_boilerplate(html: &str) -> String {
66 let mut selectors = Vec::new();
67
68 for tag in REMOVE_TAGS {
69 selectors.push(element!(tag, |el| {
70 el.remove();
71 Ok(())
72 }));
73 }
74
75 selectors.push(element!("[role='navigation']", |el| {
76 el.remove();
77 Ok(())
78 }));
79 selectors.push(element!("[role='banner']", |el| {
80 el.remove();
81 Ok(())
82 }));
83 selectors.push(element!("[role='contentinfo']", |el| {
84 el.remove();
85 Ok(())
86 }));
87 selectors.push(element!("[aria-hidden='true']", |el| {
88 el.remove();
89 Ok(())
90 }));
91 selectors.push(element!(
92 ".sidebar, .nav, .menu, .footer, .header, .ads, .ad, .advertisement",
93 |el| {
94 el.remove();
95 Ok(())
96 }
97 ));
98
99 match rewrite_str(
100 html,
101 RewriteStrSettings {
102 element_content_handlers: selectors,
103 ..RewriteStrSettings::default()
104 },
105 ) {
106 Ok(result) => result,
107 Err(_) => html.to_string(),
108 }
109}
110
111fn html_to_text(html: &str) -> String {
113 static TAG_RE: Lazy<Regex> = Lazy::new(|| Regex::new(r"<[^>]+>").expect("valid regex"));
114 static ENTITY_RE: Lazy<Regex> = Lazy::new(|| {
115 Regex::new(r"&(amp|lt|gt|quot|apos|nbsp|#\d+|#x[0-9a-fA-F]+);").expect("valid regex")
116 });
117
118 let text = html
119 .replace("<br>", "\n")
120 .replace("<br/>", "\n")
121 .replace("<br />", "\n")
122 .replace("</p>", "\n\n")
123 .replace("</div>", "\n")
124 .replace("</li>", "\n")
125 .replace("</tr>", "\n")
126 .replace("</h1>", "\n\n")
127 .replace("</h2>", "\n\n")
128 .replace("</h3>", "\n\n")
129 .replace("</h4>", "\n")
130 .replace("</h5>", "\n")
131 .replace("</h6>", "\n")
132 .replace("</blockquote>", "\n");
133
134 let text = TAG_RE.replace_all(&text, "");
135
136 let text = ENTITY_RE.replace_all(&text, |caps: ®ex::Captures| match &caps[1] {
137 "amp" => "&".to_string(),
138 "lt" => "<".to_string(),
139 "gt" => ">".to_string(),
140 "quot" => "\"".to_string(),
141 "apos" => "'".to_string(),
142 "nbsp" => " ".to_string(),
143 s if s.starts_with('#') => {
144 let num = if let Some(hex) = s.strip_prefix("#x") {
145 u32::from_str_radix(hex, 16).ok()
146 } else {
147 s[1..].parse::<u32>().ok()
148 };
149 num.and_then(char::from_u32)
150 .map(|c| c.to_string())
151 .unwrap_or_default()
152 }
153 _ => String::new(),
154 });
155
156 text.to_string()
157}
158
159fn extract_title_simple(html: &str) -> String {
161 static TITLE_RE: Lazy<Regex> =
162 Lazy::new(|| Regex::new(r"(?i)<title[^>]*>(.*?)</title>").expect("valid regex"));
163
164 TITLE_RE
165 .captures(html)
166 .and_then(|c| c.get(1))
167 .map(|m| m.as_str().trim().to_string())
168 .unwrap_or_default()
169}
170
171fn extract_metadata_simple(html: &str) -> ContentMetadata {
173 static DESC_RE: Lazy<Regex> = Lazy::new(|| {
174 Regex::new(r#"(?i)<meta[^>]*name=["']description["'][^>]*content=["']([^"']*)["']"#)
175 .expect("valid regex")
176 });
177 static AUTHOR_RE: Lazy<Regex> = Lazy::new(|| {
178 Regex::new(r#"(?i)<meta[^>]*name=["']author["'][^>]*content=["']([^"']*)["']"#)
179 .expect("valid regex")
180 });
181 static OG_DESC_RE: Lazy<Regex> = Lazy::new(|| {
182 Regex::new(r#"(?i)<meta[^>]*property=["']og:description["'][^>]*content=["']([^"']*)["']"#)
183 .expect("valid regex")
184 });
185
186 ContentMetadata {
187 description: DESC_RE
188 .captures(html)
189 .and_then(|c| c.get(1))
190 .map(|m| m.as_str().to_string())
191 .or_else(|| {
192 OG_DESC_RE
193 .captures(html)
194 .and_then(|c| c.get(1))
195 .map(|m| m.as_str().to_string())
196 }),
197 author: AUTHOR_RE
198 .captures(html)
199 .and_then(|c| c.get(1))
200 .map(|m| m.as_str().to_string()),
201 published_date: None,
202 language: None,
203 content_type: "text/html".to_string(),
204 }
205}
206
207#[cfg(test)]
208mod tests {
209 use super::*;
210
211 #[test]
212 fn extract_article_page() {
213 let html = r#"
214 <html>
215 <head><title>Article Title</title></head>
216 <body>
217 <nav><a href="/">Home</a><a href="/about">About</a></nav>
218 <script>var tracking = true;</script>
219 <article>
220 <h1>Main Heading</h1>
221 <p>First paragraph with substantial content that should
222 definitely be extracted and preserved in the output
223 because it forms the core of the article text.</p>
224 <p>Second paragraph with more details about the topic
225 that provides additional context and information
226 for the reader to understand the full picture.</p>
227 </article>
228 <footer>Copyright 2026</footer>
229 </body>
230 </html>
231 "#;
232
233 let result = extract(html, "https://example.com/article");
234 assert_eq!(result.title, "Article Title");
235 assert!(result.text.contains("First paragraph"));
236 assert!(result.text.contains("Second paragraph"));
237 assert!(!result.text.contains("tracking"));
238 assert!(!result.text.contains("Copyright"));
239 assert_eq!(result.layer_used, CepLayer::Layer2);
240 }
241
242 #[test]
243 fn html_entity_decoding() {
244 let text = html_to_text("<p>Tom & Jerry <3 "cheese"</p>");
245 assert!(text.contains("Tom & Jerry"));
246 assert!(text.contains("<3"));
247 assert!(text.contains("\"cheese\""));
248 }
249
250 #[test]
251 fn strip_scripts_and_styles() {
252 let html = r#"
253 <html><body>
254 <script>alert('xss')</script>
255 <style>.foo { color: red; }</style>
256 <p>Real content here</p>
257 </body></html>
258 "#;
259 let cleaned = strip_boilerplate(html);
260 assert!(!cleaned.contains("alert"));
261 assert!(!cleaned.contains("color: red"));
262 assert!(cleaned.contains("Real content"));
263 }
264}