servo_fetch/
extract.rs

1//! Content extraction — converts raw HTML into readable Markdown or structured JSON.
2
3use std::borrow::Cow;
4use std::fmt::Write;
5
6use dom_query::Document;
7use dom_smoothie::Readability;
8use htmd::HtmlToMarkdown;
9use serde::Serialize;
10
11use crate::layout::{self, LayoutElement};
12
13/// Errors that can occur during content extraction.
14#[derive(Debug, thiserror::Error)]
15#[non_exhaustive]
16pub enum ExtractError {
17    /// Failed to format Markdown output.
18    #[error("markdown formatting failed")]
19    Fmt(#[from] std::fmt::Error),
20    /// Failed to serialize JSON output.
21    #[error("JSON serialization failed")]
22    Json(#[from] serde_json::Error),
23}
24
25/// Structured article data for JSON output.
26#[derive(Serialize)]
27#[non_exhaustive]
28pub struct ArticleData {
29    /// Page title.
30    pub title: String,
31    /// Raw HTML content extracted by Readability.
32    pub content: String,
33    /// Readable text content (Markdown).
34    pub text_content: String,
35    /// Author or byline, if detected.
36    #[serde(skip_serializing_if = "Option::is_none")]
37    pub byline: Option<String>,
38    /// Short excerpt or description.
39    #[serde(skip_serializing_if = "Option::is_none")]
40    pub excerpt: Option<String>,
41    /// Document language (e.g. "en").
42    #[serde(skip_serializing_if = "Option::is_none")]
43    pub lang: Option<String>,
44    /// Canonical URL.
45    #[serde(skip_serializing_if = "Option::is_none")]
46    pub url: Option<String>,
47}
48
49/// Extract text content from a PDF byte slice, or an empty string on failure.
50#[must_use]
51pub fn extract_pdf(data: &[u8]) -> String {
52    match pdf_extract::extract_text_from_mem(data) {
53        Ok(text) => text,
54        Err(e) => {
55            eprintln!("warning: PDF text extraction failed: {e}");
56            String::new()
57        }
58    }
59}
60
61/// Input parameters for content extraction.
62#[non_exhaustive]
63pub struct ExtractInput<'a> {
64    /// Raw HTML of the page.
65    pub html: &'a str,
66    /// URL of the page (used for resolving relative links).
67    pub url: &'a str,
68    /// JSON-serialized layout data from the injected JS, if available.
69    pub layout_json: Option<&'a str>,
70    /// `document.body.innerText` fallback, if available.
71    pub inner_text: Option<&'a str>,
72    /// CSS selector to extract a specific section instead of using Readability.
73    pub selector: Option<&'a str>,
74}
75
76impl<'a> ExtractInput<'a> {
77    /// Create a new `ExtractInput` with required fields.
78    #[must_use]
79    pub fn new(html: &'a str, url: &'a str) -> Self {
80        Self {
81            html,
82            url,
83            layout_json: None,
84            inner_text: None,
85            selector: None,
86        }
87    }
88
89    /// Set the layout JSON data.
90    #[must_use]
91    pub fn with_layout_json(mut self, layout_json: Option<&'a str>) -> Self {
92        self.layout_json = layout_json;
93        self
94    }
95
96    /// Set the inner text fallback.
97    #[must_use]
98    pub fn with_inner_text(mut self, inner_text: Option<&'a str>) -> Self {
99        self.inner_text = inner_text;
100        self
101    }
102
103    /// Set the CSS selector for targeted extraction.
104    #[must_use]
105    pub fn with_selector(mut self, selector: Option<&'a str>) -> Self {
106        self.selector = selector;
107        self
108    }
109}
110
111/// Extract readable content as Markdown text.
112///
113/// # Errors
114/// Returns [`ExtractError::Fmt`] if Markdown assembly fails.
115pub fn extract_text(input: &ExtractInput<'_>) -> Result<String, ExtractError> {
116    if let Some(selector) = input.selector {
117        return Ok(extract_by_selector(input.html, input.layout_json, selector));
118    }
119    let article = parse_article(input.html, input.url, input.layout_json, input.inner_text);
120
121    let mut out = String::new();
122    if !article.title.is_empty() {
123        writeln!(out, "# {}\n", article.title)?;
124    }
125    if let Some(ref byline) = article.byline {
126        writeln!(out, "*{}*\n", byline.replace('*', r"\*"))?;
127    }
128    if let Some(ref excerpt) = article.excerpt {
129        writeln!(out, "> {excerpt}\n")?;
130    }
131    write!(out, "{}", article.text_content)?;
132    Ok(clean_markdown(&out))
133}
134
135/// Extract readable content as JSON.
136///
137/// # Errors
138/// Returns [`ExtractError::Json`] if JSON serialization fails.
139pub fn extract_json(input: &ExtractInput<'_>) -> Result<String, ExtractError> {
140    if let Some(selector) = input.selector {
141        let text = extract_by_selector(input.html, input.layout_json, selector);
142        let data = ArticleData {
143            title: String::new(),
144            content: String::new(),
145            text_content: text,
146            byline: None,
147            excerpt: None,
148            lang: None,
149            url: Some(input.url.to_string()),
150        };
151        return Ok(serde_json::to_string_pretty(&data)?);
152    }
153    let article = parse_article(input.html, input.url, input.layout_json, input.inner_text);
154    let data = ArticleData {
155        title: article.title,
156        content: article.content,
157        text_content: article.text_content,
158        byline: article.byline,
159        excerpt: article.excerpt,
160        lang: article.lang,
161        url: Some(input.url.to_string()),
162    };
163    Ok(serde_json::to_string_pretty(&data)?)
164}
165
166struct ParsedArticle {
167    title: String,
168    content: String,
169    text_content: String,
170    byline: Option<String>,
171    excerpt: Option<String>,
172    lang: Option<String>,
173}
174
175fn is_nextjs_error_page(text: &str) -> bool {
176    let t = text.trim();
177    t.contains("client-side exception has occurred") || t.contains("Application error: a")
178}
179
180fn parse_article(html: &str, url: &str, layout_json: Option<&str>, inner_text: Option<&str>) -> ParsedArticle {
181    let filtered = filter(html, layout_json);
182
183    let doc = Document::from(filtered.as_ref());
184    if let Ok(mut readability) = Readability::with_document(doc, Some(url), None) {
185        if let Ok(article) = readability.parse() {
186            if !is_nextjs_error_page(&article.text_content) {
187                let converter = HtmlToMarkdown::builder().build();
188                let markdown = converter
189                    .convert(&article.content)
190                    .unwrap_or_else(|_| article.text_content.to_string());
191                return ParsedArticle {
192                    title: article.title.clone(),
193                    content: article.content.to_string(),
194                    text_content: markdown,
195                    byline: article.byline.clone(),
196                    excerpt: article.excerpt.clone(),
197                    lang: article.lang,
198                };
199            }
200        }
201    }
202
203    // Readability failed or returned an error page — fall back to innerText.
204    let doc = Document::from(filtered.as_ref());
205    let title = doc.select("title").text().to_string();
206    let body_text = inner_text.filter(|s| !s.trim().is_empty()).map_or_else(
207        || {
208            eprintln!(
209                "warning: could not extract content. \
210                 Try --js \"document.body.innerText\" for JS-heavy sites."
211            );
212            String::new()
213        },
214        String::from,
215    );
216    ParsedArticle {
217        title,
218        content: String::new(),
219        text_content: body_text,
220        byline: None,
221        excerpt: None,
222        lang: None,
223    }
224}
225
226fn extract_by_selector(html: &str, layout_json: Option<&str>, selector: &str) -> String {
227    let filtered = filter(html, layout_json);
228    let doc = Document::from(filtered.as_ref());
229    let selected = doc.select(selector);
230    let fragment = selected.html();
231    if fragment.is_empty() {
232        return String::new();
233    }
234    let converter = HtmlToMarkdown::builder().skip_tags(vec!["script", "style"]).build();
235    let markdown = converter
236        .convert(&fragment)
237        .unwrap_or_else(|_| selected.text().to_string());
238    clean_markdown(&markdown)
239}
240
241fn filter<'a>(html: &'a str, layout_json: Option<&str>) -> Cow<'a, str> {
242    layout_json
243        .and_then(|lj| serde_json::from_str::<Vec<LayoutElement>>(lj).ok())
244        .map_or(Cow::Borrowed(html), |els| {
245            let sels = layout::selectors_to_strip(&els);
246            if sels.is_empty() {
247                return Cow::Borrowed(html);
248            }
249            let doc = Document::from(html);
250            for sel in &sels {
251                doc.select(sel).remove();
252            }
253            Cow::Owned(doc.html().to_string())
254        })
255}
256
257// Collapse runs of 3+ blank lines down to 2.
258fn clean_markdown(input: &str) -> String {
259    let mut result = String::with_capacity(input.len());
260    let mut blank_count = 0u8;
261    for line in input.lines() {
262        if line.trim().is_empty() {
263            blank_count = blank_count.saturating_add(1);
264            if blank_count <= 2 {
265                result.push('\n');
266            }
267        } else {
268            blank_count = 0;
269            result.push_str(line);
270            result.push('\n');
271        }
272    }
273    result
274}
275
276#[cfg(test)]
277mod tests {
278    use super::*;
279
280    #[test]
281    fn is_nextjs_error_page_detects_nextjs() {
282        assert!(is_nextjs_error_page(
283            "Application error: a client-side exception has occurred"
284        ));
285    }
286
287    #[test]
288    fn is_nextjs_error_page_ignores_normal_content() {
289        assert!(!is_nextjs_error_page("This article discusses error handling in Rust."));
290        assert!(!is_nextjs_error_page(
291            "A long page about many topics that happens to mention errors somewhere in the middle of a paragraph."
292        ));
293    }
294
295    #[test]
296    fn clean_markdown_collapses_blank_lines() {
297        let input = "line1\n\n\n\n\nline2\n";
298        let result = clean_markdown(input);
299        assert_eq!(result, "line1\n\n\nline2\n");
300    }
301
302    #[test]
303    fn clean_markdown_preserves_single_blank() {
304        let input = "a\n\nb\n";
305        assert_eq!(clean_markdown(input), "a\n\nb\n");
306    }
307
308    #[test]
309    fn filter_without_layout_returns_original() {
310        let html = "<html><body>hello</body></html>";
311        let result = filter(html, None);
312        assert_eq!(result.as_ref(), html);
313    }
314
315    #[test]
316    fn filter_strips_footer() {
317        let html = r#"<html><body><footer style="position:static">nav</footer><p>content</p></body></html>"#;
318        let layout = r#"[{"tag":"FOOTER","role":null,"w":1280,"h":100,"position":"static"}]"#;
319        let result = filter(html, Some(layout));
320        assert!(!result.contains("<footer"));
321        assert!(result.contains("content"));
322    }
323
324    #[test]
325    fn extract_input_builder() {
326        let input = ExtractInput::new("<html></html>", "https://example.com")
327            .with_layout_json(Some("[]"))
328            .with_inner_text(Some("hello"))
329            .with_selector(Some("article"));
330        assert_eq!(input.layout_json, Some("[]"));
331        assert_eq!(input.inner_text, Some("hello"));
332        assert_eq!(input.selector, Some("article"));
333    }
334}
servo_fetch/extract.rs

servo_fetch/
extract.rs
