servo_fetch/
extract.rs

1//! Content extraction — converts raw HTML into readable Markdown or structured JSON.
2
3use std::borrow::Cow;
4use std::collections::HashMap;
5use std::fmt::{self, Write as _};
6
7use dom_query::Document;
8use dom_smoothie::Readability;
9use htmd::HtmlToMarkdown;
10use serde::Serialize;
11use servo::accesskit::{Node, NodeId};
12
13use crate::layout::{self, LayoutElement};
14use crate::visibility::{self, A11yIndex, VisibilityPolicy};
15
16/// Errors that can occur during content extraction.
17#[derive(Debug, thiserror::Error)]
18#[non_exhaustive]
19pub enum ExtractError {
20    /// Failed to format Markdown output.
21    #[error("markdown formatting failed")]
22    Fmt(#[from] fmt::Error),
23    /// Failed to serialize JSON output.
24    #[error("JSON serialization failed")]
25    Json(#[from] serde_json::Error),
26    /// The provided CSS selector is invalid.
27    #[error("invalid CSS selector")]
28    InvalidSelector,
29}
30
31/// Structured article data for JSON output.
32#[derive(Debug, Clone, Serialize)]
33#[serde(rename_all = "camelCase")]
34pub struct ArticleData {
35    /// Page title.
36    pub title: String,
37    /// Raw HTML content extracted by Readability.
38    pub content: String,
39    /// Readable text content (Markdown).
40    pub text_content: String,
41    /// Author or byline, if detected.
42    #[serde(skip_serializing_if = "Option::is_none")]
43    pub byline: Option<String>,
44    /// Short excerpt or description.
45    #[serde(skip_serializing_if = "Option::is_none")]
46    pub excerpt: Option<String>,
47    /// Document language (e.g. "en").
48    #[serde(skip_serializing_if = "Option::is_none")]
49    pub lang: Option<String>,
50    /// Canonical URL.
51    #[serde(skip_serializing_if = "Option::is_none")]
52    pub url: Option<String>,
53}
54
55/// Extract text content from a PDF byte slice, or an empty string on failure.
56#[must_use]
57pub fn extract_pdf(data: &[u8]) -> String {
58    match pdf_extract::extract_text_from_mem(data) {
59        Ok(text) => text,
60        Err(e) => {
61            tracing::warn!(error = %e, "PDF text extraction failed");
62            String::new()
63        }
64    }
65}
66
67/// Input parameters for content extraction.
68#[non_exhaustive]
69pub struct ExtractInput<'a> {
70    /// Raw HTML of the page.
71    pub html: &'a str,
72    /// URL of the page (used for resolving relative links).
73    pub url: &'a str,
74    /// JSON-serialized layout data from the injected JS, if available.
75    pub layout_json: Option<&'a str>,
76    /// JSON-serialized visibility data from the injected JS, if available.
77    pub visibility_json: Option<&'a str>,
78    /// AccessKit accessibility tree, if available.
79    pub a11y: Option<&'a HashMap<NodeId, Node>>,
80    /// `document.body.innerText` fallback, if available.
81    pub inner_text: Option<&'a str>,
82    /// CSS selector to extract a specific section instead of using Readability.
83    pub selector: Option<&'a str>,
84    /// Visibility policy controlling which hidden content is stripped.
85    pub visibility: VisibilityPolicy,
86}
87
88impl<'a> ExtractInput<'a> {
89    /// Create a new `ExtractInput` with required fields.
90    #[must_use]
91    pub fn new(html: &'a str, url: &'a str) -> Self {
92        Self {
93            html,
94            url,
95            layout_json: None,
96            visibility_json: None,
97            a11y: None,
98            inner_text: None,
99            selector: None,
100            visibility: VisibilityPolicy::default(),
101        }
102    }
103
104    /// Set the layout JSON data.
105    #[must_use]
106    pub fn with_layout_json(mut self, layout_json: Option<&'a str>) -> Self {
107        self.layout_json = layout_json;
108        self
109    }
110
111    /// Set the visibility JSON data.
112    #[must_use]
113    pub fn with_visibility_json(mut self, visibility_json: Option<&'a str>) -> Self {
114        self.visibility_json = visibility_json;
115        self
116    }
117
118    /// Set the typed accessibility tree.
119    #[must_use]
120    pub fn with_a11y(mut self, a11y: Option<&'a HashMap<NodeId, Node>>) -> Self {
121        self.a11y = a11y;
122        self
123    }
124
125    /// Set the inner text fallback.
126    #[must_use]
127    pub fn with_inner_text(mut self, inner_text: Option<&'a str>) -> Self {
128        self.inner_text = inner_text;
129        self
130    }
131
132    /// Set the CSS selector for targeted extraction.
133    #[must_use]
134    pub fn with_selector(mut self, selector: Option<&'a str>) -> Self {
135        self.selector = selector;
136        self
137    }
138
139    /// Set the visibility policy.
140    #[must_use]
141    pub fn with_visibility(mut self, policy: VisibilityPolicy) -> Self {
142        self.visibility = policy;
143        self
144    }
145}
146
147/// Extract readable content as Markdown text.
148pub fn extract_text(input: &ExtractInput<'_>) -> Result<String, ExtractError> {
149    if let Some(selector) = input.selector {
150        return extract_by_selector(input, selector);
151    }
152    let article = parse_article(input);
153
154    let mut out = String::new();
155    if !article.title.is_empty() {
156        writeln!(out, "# {}\n", article.title)?;
157    }
158    if let Some(ref byline) = article.byline {
159        writeln!(out, "*{}*\n", byline.replace('*', r"\*"))?;
160    }
161    if let Some(ref excerpt) = article.excerpt {
162        writeln!(out, "> {excerpt}\n")?;
163    }
164    write!(out, "{}", article.text_content)?;
165    Ok(clean_markdown(&out))
166}
167
168/// Extract readable content as structured article data.
169pub fn extract_article(input: &ExtractInput<'_>) -> Result<ArticleData, ExtractError> {
170    if let Some(selector) = input.selector {
171        let text = extract_by_selector(input, selector)?;
172        return Ok(ArticleData {
173            title: String::new(),
174            content: String::new(),
175            text_content: text,
176            byline: None,
177            excerpt: None,
178            lang: None,
179            url: Some(input.url.to_string()),
180        });
181    }
182    let article = parse_article(input);
183    Ok(ArticleData {
184        title: article.title,
185        content: article.content,
186        text_content: article.text_content,
187        byline: article.byline,
188        excerpt: article.excerpt,
189        lang: article.lang,
190        url: Some(input.url.to_string()),
191    })
192}
193
194/// Extract readable content as JSON.
195pub fn extract_json(input: &ExtractInput<'_>) -> Result<String, ExtractError> {
196    Ok(serde_json::to_string_pretty(&extract_article(input)?)?)
197}
198
199struct ParsedArticle {
200    title: String,
201    content: String,
202    text_content: String,
203    byline: Option<String>,
204    excerpt: Option<String>,
205    lang: Option<String>,
206}
207
208fn is_nextjs_error_page(text: &str) -> bool {
209    let t = text.trim();
210    t.contains("client-side exception has occurred") || t.contains("Application error: a")
211}
212
213fn parse_article(input: &ExtractInput<'_>) -> ParsedArticle {
214    let filtered = filter(input);
215
216    let doc = Document::from(filtered.as_ref());
217    if let Ok(mut readability) = Readability::with_document(doc, Some(input.url), None) {
218        if let Ok(article) = readability.parse() {
219            if !is_nextjs_error_page(&article.text_content) {
220                let converter = HtmlToMarkdown::builder().build();
221                let markdown = converter
222                    .convert(&article.content)
223                    .unwrap_or_else(|_| article.text_content.to_string());
224                return ParsedArticle {
225                    title: article.title.clone(),
226                    content: article.content.to_string(),
227                    text_content: markdown,
228                    byline: article.byline.clone(),
229                    excerpt: article.excerpt.clone(),
230                    lang: article.lang,
231                };
232            }
233        }
234    }
235
236    // Readability failed or returned an error page — fall back to the filtered
237    // document's text content.
238    let doc = Document::from(filtered.as_ref());
239    doc.select("script, style, noscript").remove();
240    let title = doc.select("title").text().to_string();
241    let filtered_text = doc.select("body").text().to_string();
242    let body_text = if filtered_text.trim().is_empty() {
243        input.inner_text.filter(|s| !s.trim().is_empty()).map_or_else(
244            || {
245                tracing::warn!(r#"could not extract content; try --js "document.body.innerText" for JS-heavy sites"#);
246                String::new()
247            },
248            String::from,
249        )
250    } else {
251        filtered_text
252    };
253    ParsedArticle {
254        title,
255        content: String::new(),
256        text_content: body_text,
257        byline: None,
258        excerpt: None,
259        lang: None,
260    }
261}
262
263fn extract_by_selector(input: &ExtractInput<'_>, selector: &str) -> Result<String, ExtractError> {
264    let matcher = dom_query::Matcher::new(selector).map_err(|_| ExtractError::InvalidSelector)?;
265    let filtered = filter(input);
266    let doc = Document::from(filtered.as_ref());
267    let selected = doc.select_matcher(&matcher);
268    let fragment = selected.html();
269    if fragment.is_empty() {
270        return Ok(String::new());
271    }
272    let converter = HtmlToMarkdown::builder().skip_tags(vec!["script", "style"]).build();
273    let markdown = converter
274        .convert(&fragment)
275        .unwrap_or_else(|_| selected.text().to_string());
276    Ok(clean_markdown(&markdown))
277}
278
279fn filter<'a>(input: &'a ExtractInput<'a>) -> Cow<'a, str> {
280    let mut selectors: Vec<String> = Vec::new();
281
282    if let Some(lj) = input.layout_json
283        && let Ok(els) = serde_json::from_str::<Vec<LayoutElement>>(lj)
284    {
285        selectors.extend(layout::selectors_to_strip(&els));
286    }
287
288    let a11y_index = input.a11y.map(A11yIndex::new);
289
290    selectors.extend(visibility::selectors_to_strip(
291        input.visibility,
292        a11y_index.as_ref(),
293        input.visibility_json,
294    ));
295
296    let needs_attr_cleanup = input.visibility_json.is_some() || input.html.contains("data-vf-id=");
297    if selectors.is_empty() && !needs_attr_cleanup {
298        return Cow::Borrowed(input.html);
299    }
300
301    let doc = Document::from(input.html);
302    for sel in &selectors {
303        doc.select(sel).remove();
304    }
305    if needs_attr_cleanup {
306        doc.select("[data-vf-id]").remove_attr("data-vf-id");
307    }
308    Cow::Owned(doc.html().to_string())
309}
310
311// Collapse runs of 3+ blank lines down to 2.
312fn clean_markdown(input: &str) -> String {
313    let mut result = String::with_capacity(input.len());
314    let mut blank_count = 0u8;
315    for line in input.lines() {
316        if line.trim().is_empty() {
317            blank_count = blank_count.saturating_add(1);
318            if blank_count <= 2 {
319                result.push('\n');
320            }
321        } else {
322            blank_count = 0;
323            result.push_str(line);
324            result.push('\n');
325        }
326    }
327    result
328}
329
330#[cfg(test)]
331mod tests {
332    use super::*;
333
334    #[test]
335    fn is_nextjs_error_page_detects_nextjs() {
336        assert!(is_nextjs_error_page(
337            "Application error: a client-side exception has occurred"
338        ));
339    }
340
341    #[test]
342    fn is_nextjs_error_page_ignores_normal_content() {
343        assert!(!is_nextjs_error_page("This article discusses error handling in Rust."));
344        assert!(!is_nextjs_error_page(
345            "A long page about many topics that happens to mention errors somewhere in the middle of a paragraph."
346        ));
347    }
348
349    #[test]
350    fn clean_markdown_collapses_blank_lines() {
351        let input = "line1\n\n\n\n\nline2\n";
352        let result = clean_markdown(input);
353        assert_eq!(result, "line1\n\n\nline2\n");
354    }
355
356    #[test]
357    fn clean_markdown_preserves_single_blank() {
358        let input = "a\n\nb\n";
359        assert_eq!(clean_markdown(input), "a\n\nb\n");
360    }
361
362    #[test]
363    fn filter_off_policy_keeps_visible_content() {
364        let input = ExtractInput::new("<html><body>hello</body></html>", "").with_visibility(VisibilityPolicy::off());
365        let result = filter(&input);
366        assert!(result.contains("hello"));
367    }
368
369    #[test]
370    fn filter_strips_footer() {
371        let html = r#"<html><body><footer style="position:static">nav</footer><p>content</p></body></html>"#;
372        let layout = r#"[{"tag":"FOOTER","role":null,"w":1280,"h":100,"position":"static"}]"#;
373        let input = ExtractInput::new(html, "")
374            .with_layout_json(Some(layout))
375            .with_visibility(VisibilityPolicy::off());
376        let result = filter(&input);
377        assert!(!result.contains("<footer"));
378        assert!(result.contains("content"));
379    }
380
381    #[test]
382    fn filter_strips_visibility_flagged_element() {
383        let html = r#"<html><body><p data-vf-id="1">drop</p><p data-vf-id="2">keep</p></body></html>"#;
384        let visibility = r#"[{"id":"1","flags":16}]"#;
385        let input = ExtractInput::new(html, "")
386            .with_visibility_json(Some(visibility))
387            .with_visibility(VisibilityPolicy::moderate());
388        let result = filter(&input);
389        assert!(!result.contains("drop"));
390        assert!(result.contains("keep"));
391    }
392
393    #[test]
394    fn filter_removes_data_vf_id_from_output() {
395        let html = r#"<html><body><p data-vf-id="1">keep</p></body></html>"#;
396        let input = ExtractInput::new(html, "")
397            .with_layout_json(Some("[]"))
398            .with_visibility(VisibilityPolicy::off());
399        let result = filter(&input);
400        assert!(!result.contains("data-vf-id"));
401    }
402
403    #[test]
404    fn extract_input_builder() {
405        let input = ExtractInput::new("<html></html>", "https://example.com")
406            .with_layout_json(Some("[]"))
407            .with_visibility_json(Some(r"[]"))
408            .with_inner_text(Some("hello"))
409            .with_selector(Some("article"))
410            .with_visibility(VisibilityPolicy::strict());
411        assert_eq!(input.layout_json, Some("[]"));
412        assert_eq!(input.visibility_json, Some("[]"));
413        assert_eq!(input.inner_text, Some("hello"));
414        assert_eq!(input.selector, Some("article"));
415    }
416
417    #[test]
418    fn clean_markdown_no_trailing_newline() {
419        let input = "line1\nline2";
420        let result = clean_markdown(input);
421        assert_eq!(result, "line1\nline2\n");
422    }
423}
servo_fetch/extract.rs

servo_fetch/
extract.rs
